refactor memory management code

We now support immortal objects, which the GC will scan for references
but not consider for collection.  On x86_64, we allocate JIT code memory
via mmap, which lets us map memory into the bottom 2GB of the address
space, ensuring that 32-bit relative jumps and calls work.

makefile

@@ -134,6 +134,7 @@ generated-code = \
 
 interpreter-depends = \
 	$(generated-code) \
+	$(src)/allocator.h \
 	$(src)/common.h \
 	$(src)/system.h \
 	$(src)/heap.h \
@@ -144,7 +145,8 @@ interpreter-depends = \
 	$(src)/constants.h \
 	$(src)/jnienv.h \
 	$(src)/machine.h \
-	$(src)/util.h
+	$(src)/util.h \
+	$(src)/zone.h
 
 interpreter-sources = \
 	$(src)/$(system).cpp \

src/allocator.h

@@ -0,0 +1,18 @@
+#ifndef ALLOCATOR_H
+#define ALLOCATOR_H
+
+#include "common.h"
+
+namespace vm {
+
+class Allocator {
+ public:
+  virtual ~Allocator() { }
+  virtual void* tryAllocate(unsigned size) = 0;
+  virtual void* allocate(unsigned size) = 0;
+  virtual void free(const void* p, unsigned size) = 0;
+};
+
+} // namespace vm
+
+#endif//ALLOCATOR_H

src/builtin.cpp

@@ -126,7 +126,7 @@ Java_java_lang_ClassLoader_defineClass
   uint8_t* buffer = static_cast<uint8_t*>(t->m->system->allocate(length));
   memcpy(buffer, &byteArrayBody(t, *b, offset), length);
   object c = parseClass(t, buffer, length);
-  t->m->system->free(buffer);
+  t->m->system->free(buffer, length);
   return makeLocalReference(t, c);
 }
 

src/compile.cpp

@@ -436,7 +436,7 @@ class Context {
 
   Context(MyThread* t, object method, uint8_t* indirectCaller):
     t(t),
-    zone(t->m->system, 16 * 1024),
+    zone(t->m->system, t->m->system, 16 * 1024),
     c(makeCompiler(t->m->system, &zone, indirectCaller)),
     method(method),
     objectPool(0),
@@ -449,7 +449,7 @@ class Context {
 
   Context(MyThread* t):
     t(t),
-    zone(t->m->system, 256),
+    zone(t->m->system, t->m->system, LikelyPageSizeInBytes),
     c(makeCompiler(t->m->system, &zone, 0)),
     method(0),
     objectPool(0),
@@ -3629,6 +3629,9 @@ calculateFrameMaps(MyThread* t, Context* context)
   updateTraceElements(t, context, roots, 0);
 }
 
+Allocator*
+codeAllocator(MyThread* t);
+
 object
 finish(MyThread* t, Context* context, const char* name)
 {
@@ -3636,9 +3639,11 @@ finish(MyThread* t, Context* context, const char* name)
 
   unsigned count = ceiling(c->codeSize() + c->poolSize(), BytesPerWord);
   unsigned size = count + singletonMaskSize(count);
-  object result = allocate2
+  object result = allocate3
-    (t, SingletonBody + size * BytesPerWord, true, true);
+    (t, codeAllocator(t), Machine::ImmortalAllocation,
-  initSingleton(t, result, size, true); 
+     SingletonBody + (size * BytesPerWord), true);
+  initSingleton(t, result, size, true);
+  mark(t, result, 0);
   singletonMask(t, result)[0] = 1;
 
   uint8_t* start = reinterpret_cast<uint8_t*>(&singletonValue(t, result, 0));
@@ -4326,7 +4331,9 @@ class MyProcessor: public Processor {
     nativeCompiled(0),
     addressTable(0),
     addressCount(0),
-    indirectCaller(0)
+    indirectCaller(0),
+    indirectCallerSize(0),
+    codeAllocator(s, s->codeAllocator(), 64 * 1024)
   { }
 
   virtual Thread*
@@ -4576,19 +4583,23 @@ class MyProcessor: public Processor {
   virtual void dispose(Thread* vmt) {
     MyThread* t = static_cast<MyThread*>(vmt);
 
-    t->m->system->handleSegFault(0);
-
     while (t->reference) {
       vm::dispose(t, t->reference);
     }
+
+    s->free(t, sizeof(*t));
   }
 
   virtual void dispose() {
+    codeAllocator.dispose();
+    
+    s->handleSegFault(0);
+
     if (indirectCaller) {
-      s->free(indirectCaller);
+      s->free(indirectCaller, indirectCallerSize);
     }
 
-    s->free(this);
+    s->free(this, sizeof(*this));
   }
   
   System* s;
@@ -4597,7 +4608,9 @@ class MyProcessor: public Processor {
   object addressTable;
   unsigned addressCount;
   uint8_t* indirectCaller;
+  unsigned indirectCallerSize;
   SegFaultHandler segFaultHandler;
+  Zone codeAllocator;
 };
 
 MyProcessor*
@@ -4618,8 +4631,9 @@ processor(MyThread* t)
 
       c->jmp(c->indirectTarget());
 
+      p->indirectCallerSize = c->codeSize();
       p->indirectCaller = static_cast<uint8_t*>
-        (t->m->system->allocate(c->codeSize()));
+        (t->m->system->allocate(p->indirectCallerSize));
       c->writeTo(p->indirectCaller);
 
       if (Verbose) {
@@ -4758,6 +4772,11 @@ insertTraceNode(MyThread* t, object node)
   set(t, p->addressTable, ArrayBody + (index * BytesPerWord), node);
 }
 
+Allocator*
+codeAllocator(MyThread* t) {
+  return &(processor(t)->codeAllocator);
+}
+
 } // namespace
 
 namespace vm {

src/compiler.cpp

@@ -160,7 +160,6 @@ class Context {
     plan.dispose();
     code.dispose();
     constantPool.dispose();
-    if (segmentTable) s->free(segmentTable);
   }
 
   System* s;
@@ -2268,7 +2267,8 @@ writeCode(Context* c)
   unsigned tableSize = (c->plan.length() / BytesPerWord);
 
   if (c->codeLength < 0) {
-    c->segmentTable = static_cast<Segment**>(c->s->allocate(c->plan.length()));
+    c->segmentTable = static_cast<Segment**>
+      (c->zone->allocate(c->plan.length()));
     
     for (unsigned i = 0; i < tableSize; ++i) {
       c->plan.get(i * BytesPerWord, c->segmentTable + i, BytesPerWord);

src/finder.cpp

@@ -7,12 +7,14 @@ using namespace vm;
 namespace {
 
 const char*
-append(System* s, const char* a, const char* b, const char* c)
+append(System* s, unsigned* length, const char* a, const char* b,
+       const char* c)
 {
   unsigned al = strlen(a);
   unsigned bl = strlen(b);
   unsigned cl = strlen(c);
-  char* p = static_cast<char*>(s->allocate(al + bl + cl + 1));
+  *length = al + bl + cl;
+  char* p = static_cast<char*>(s->allocate(*length + 1));
   memcpy(p, a, al);
   memcpy(p + al, b, bl);
   memcpy(p + al + bl, c, cl + 1);
@@ -20,10 +22,11 @@ append(System* s, const char* a, const char* b, const char* c)
 }
 
 const char*
-copy(System* s, const char* a)
+copy(System* s, unsigned* length, const char* a)
 {
   unsigned al = strlen(a);
-  char* p = static_cast<char*>(s->allocate(al + 1));
+  *length = al;
+  char* p = static_cast<char*>(s->allocate(*length + 1));
   memcpy(p, a, al + 1);
   return p;
 }
@@ -51,15 +54,16 @@ class Element {
 
 class DirectoryElement: public Element {
  public:
-  DirectoryElement(System* s, const char* name):
+  DirectoryElement(System* s, const char* name, unsigned nameLength):
-    s(s), name(name)
+    s(s), name(name), nameLength(nameLength)
   { }
 
   virtual System::Region* find(const char* name) {
-    const char* file = append(s, this->name, "/", name);
+    unsigned length;
+    const char* file = append(s, &length, this->name, "/", name);
     System::Region* region;
     System::Status status = s->map(&region, file);
-    s->free(file);
+    s->free(file, length + 1);
 
     if (s->success(status)) {
       return region;
@@ -69,19 +73,21 @@ class DirectoryElement: public Element {
   }
 
   virtual bool exists(const char* name)  {
-    const char* file = append(s, this->name, "/", name);
+    unsigned length;
+    const char* file = append(s, &length, this->name, "/", name);
     System::FileType type = s->identify(file);
-    s->free(file);
+    s->free(file, length + 1);
     return type != System::DoesNotExist;
   }
 
   virtual void dispose() {
-    s->free(name);
+    s->free(name, nameLength + 1);
-    s->free(this);
+    s->free(this, sizeof(*this));
   }
 
   System* s;
   const char* name;
+  unsigned nameLength;
 };
 
 class PointerRegion: public System::Region {
@@ -101,7 +107,7 @@ class PointerRegion: public System::Region {
   }
 
   virtual void dispose() {
-    s->free(this);
+    s->free(this, sizeof(*this));
   }
 
   System* s;
@@ -125,7 +131,7 @@ class DataRegion: public System::Region {
   }
 
   virtual void dispose() {
-    s->free(this);
+    s->free(this, sizeof(*this) + length_);
   }
 
   System* s;
@@ -311,8 +317,8 @@ class JarIndex {
   }
 
   void dispose() {
-    s->free(nodes);
+    s->free(nodes, sizeof(Node) * capacity);
-    s->free(this);
+    s->free(this, sizeof(*this) + (sizeof(Node*) * capacity));
   }
 
   System* s;
@@ -324,8 +330,8 @@ class JarIndex {
 
 class JarElement: public Element {
  public:
-  JarElement(System* s, const char* name):
+  JarElement(System* s, const char* name, unsigned nameLength):
-    s(s), name(name), index(0)
+    s(s), name(name), nameLength(nameLength), index(0)
   { }
 
   virtual void init() {
@@ -355,26 +361,27 @@ class JarElement: public Element {
   }
 
   virtual void dispose() {
-    s->free(name);
+    s->free(name, nameLength + 1);
     if (index) {
       index->dispose();
     }
     if (region) {
       region->dispose();
     }
-    s->free(this);
+    s->free(this, sizeof(*this));
   }
 
   System* s;
   const char* name;
+  unsigned nameLength;
   System::Region* region;
   JarIndex* index;
 };
 
 class BuiltinElement: public JarElement {
  public:
-  BuiltinElement(System* s, const char* name):
+  BuiltinElement(System* s, const char* name, unsigned nameLength):
-    JarElement(s, name)
+    JarElement(s, name, nameLength)
   { }
 
   virtual void init() {
@@ -441,7 +448,8 @@ parsePath(System* s, const char* path)
       memcpy(name, token.s + 1, token.length - 1);
       name[token.length - 2] = 0; 
   
-      e = new (s->allocate(sizeof(BuiltinElement))) BuiltinElement(s, name);
+      e = new (s->allocate(sizeof(BuiltinElement)))
+        BuiltinElement(s, name, token.length - 2);
     } else {
       char* name = static_cast<char*>(s->allocate(token.length + 1));
       memcpy(name, token.s, token.length);
@@ -449,16 +457,17 @@ parsePath(System* s, const char* path)
 
       switch (s->identify(name)) {
       case System::File: {
-        e = new (s->allocate(sizeof(JarElement))) JarElement(s, name);
+        e = new (s->allocate(sizeof(JarElement)))
+          JarElement(s, name, token.length);
       } break;
 
       case System::Directory: {
         e = new (s->allocate(sizeof(DirectoryElement)))
-          DirectoryElement(s, name);
+          DirectoryElement(s, name, token.length);
       } break;
 
       default: {
-        s->free(name);
+        s->free(name, token.length + 1);
         e = 0;
       } break;
       }
@@ -482,7 +491,7 @@ class MyFinder: public Finder {
   MyFinder(System* system, const char* path):
     system(system),
     path_(parsePath(system, path)),
-    pathString(copy(system, path))
+    pathString(copy(system, &pathStringLength, path))
   { }
 
   virtual System::Region* find(const char* name) {
@@ -516,13 +525,14 @@ class MyFinder: public Finder {
       e = e->next;
       t->dispose();
     }
-    system->free(pathString);
+    system->free(pathString, pathStringLength + 1);
-    system->free(this);
+    system->free(this, sizeof(*this));
   }
 
   System* system;
   Element* path_;
   const char* pathString;
+  unsigned pathStringLength;
 };
 
 } // namespace

src/heap.cpp

@@ -285,7 +285,7 @@ class Segment {
       while (data == 0) {
         data = static_cast<uintptr_t*>
           (system(context)->tryAllocate
-           ((capacity_ + map->footprint(capacity_)) * BytesPerWord));
+           ((capacity_ + mapFootprint(capacity_)) * BytesPerWord));
 
         if (data == 0) {
           if (capacity_ > minimum) {
@@ -305,6 +305,10 @@ class Segment {
     }
   }
 
+  unsigned mapFootprint(unsigned capacity) {
+    return map and capacity ? map->footprint(capacity) : 0;
+  }
+
   unsigned capacity() {
     return capacity_;
   }
@@ -318,7 +322,10 @@ class Segment {
   }
 
   void replaceWith(Segment* s) {
-    if (data) system(context)->free(data);
+    if (data) {
+      system(context)->free
+        (data, (capacity() + mapFootprint(capacity())) * BytesPerWord);
+    }
     data = s->data;
     s->data = 0;
 
@@ -368,7 +375,8 @@ class Segment {
   }
 
   void dispose() {
-    system(context)->free(data);
+    system(context)->free
+      (data, (capacity() + mapFootprint(capacity())) * BytesPerWord);
     data = 0;
     map = 0;
   }
@@ -376,28 +384,37 @@ class Segment {
 
 class Fixie {
  public:
-  Fixie(unsigned size, bool hasMask, Fixie** handle):
+  Fixie(unsigned size, bool hasMask, Fixie** handle, bool immortal):
-    age(0),
+    age(immortal ? FixieTenureThreshold + 1 : 0),
     hasMask(hasMask),
     marked(false),
-    dirty(false)
+    dirty(false),
+    size(size)
   {
-    memset(mask(size), 0, maskSize(size, hasMask));
+    memset(mask(), 0, maskSize(size, hasMask));
     add(handle);
     if (DebugFixies) {
-      fprintf(stderr, "make fixie %p\n", this);
+      fprintf(stderr, "make fixie %p of size %d\n", this, totalSize());
     }
   }
 
+  bool immortal() {
+    return age == FixieTenureThreshold + 1;
+  }
+
   void add(Fixie** handle) {
     this->handle = handle;
-    next = *handle;
+    if (handle) {
-    if (next) next->handle = &next;
+      next = *handle;
-    *handle = this;
+      if (next) next->handle = &next;
+      *handle = this;
+    } else {
+      next = 0;
+    }
   }
 
   void remove() {
-    *handle = next;
+    if (handle) *handle = next;
     if (next) next->handle = handle;
   }
 
@@ -414,7 +431,7 @@ class Fixie {
     return static_cast<void**>(static_cast<void*>(body_));
   }
 
-  uintptr_t* mask(unsigned size) {
+  uintptr_t* mask() {
     return body_ + size;
   }
 
@@ -426,7 +443,7 @@ class Fixie {
     return sizeof(Fixie) + (size * BytesPerWord) + maskSize(size, hasMask);
   }
 
-  unsigned totalSize(unsigned size) {
+  unsigned totalSize() {
     return totalSize(size, hasMask);
   }
 
@@ -434,6 +451,7 @@ class Fixie {
   bool hasMask;
   bool marked;
   bool dirty;
+  unsigned size;
   Fixie* next;
   Fixie** handle;
   uintptr_t body_[0];
@@ -485,7 +503,7 @@ class Context {
 
     fixies(0),
     tenuredFixies(0),
-    dirtyFixies(0),
+    dirtyTenuredFixies(0),
     markedFixies(0),
     visitedFixies(0),
 
@@ -500,7 +518,7 @@ class Context {
     gen2.dispose();
     nextGen2.dispose();
     free(this, &tenuredFixies);
-    free(this, &dirtyFixies);
+    free(this, &dirtyTenuredFixies);
     free(this, &fixies);
     client->dispose();
   }
@@ -539,7 +557,7 @@ class Context {
 
   Fixie* fixies;
   Fixie* tenuredFixies;
-  Fixie* dirtyFixies;
+  Fixie* dirtyTenuredFixies;
   Fixie* markedFixies;
   Fixie* visitedFixies;
 
@@ -681,11 +699,15 @@ free(Context* c, Fixie** fixies)
 {
   for (Fixie** p = fixies; *p;) {
     Fixie* f = *p;
-    *p = f->next;
+    if (f->immortal()) {
-    if (DebugFixies) {
+      p = &(f->next);
-      fprintf(stderr, "free fixie %p\n", f);
+    } else {
+      *p = f->next;
+      if (DebugFixies) {
+        fprintf(stderr, "free fixie %p\n", f);
+      }
+      c->system->free(f, f->totalSize());
     }
-    c->system->free(f);
   }
 }
 
@@ -696,7 +718,7 @@ sweepFixies(Context* c)
 
   if (c->mode == Heap::MajorCollection) {
     free(c, &(c->tenuredFixies));
-    free(c, &(c->dirtyFixies));
+    free(c, &(c->dirtyTenuredFixies));
 
     c->tenuredFixieFootprint = 0;
   }
@@ -708,29 +730,31 @@ sweepFixies(Context* c)
     Fixie* f = *p;
     *p = f->next;
 
-    unsigned size = c->client->sizeInWords(f->body());
+    if (not f->immortal()) {
-
+      ++ f->age;
-    ++ f->age;
+      if (f->age > FixieTenureThreshold) {
-    if (f->age > FixieTenureThreshold) {
+        f->age = FixieTenureThreshold;
-      f->age = FixieTenureThreshold;
+      } else if (static_cast<unsigned>(f->age + 1) == FixieTenureThreshold) {
-    } else if (static_cast<unsigned>(f->age + 1) == FixieTenureThreshold) {
+        c->fixieTenureFootprint += f->totalSize();
-      c->fixieTenureFootprint += f->totalSize(size);
+      }
     }
 
-    if (f->age == FixieTenureThreshold) {
+    if (f->age >= FixieTenureThreshold) {
       if (DebugFixies) {
         fprintf(stderr, "tenure fixie %p (dirty: %d)\n", f, f->dirty);
       }
 
-      c->tenuredFixieFootprint += size;
+      if (not f->immortal()) {
+        c->tenuredFixieFootprint += f->totalSize();
+      }
 
       if (f->dirty) {
-        f->move(&(c->dirtyFixies));
+        f->move(&(c->dirtyTenuredFixies));
       } else {
         f->move(&(c->tenuredFixies));
       }
     } else {
-      c->untenuredFixieFootprint += size;
+      c->untenuredFixieFootprint += f->totalSize();
 
       f->move(&(c->fixies));
     }
@@ -740,7 +764,7 @@ sweepFixies(Context* c)
 
   c->tenuredFixieCeiling = max
     (c->tenuredFixieFootprint * 2,
-     InitialTenuredFixieCeilingInBytes / BytesPerWord);
+     InitialTenuredFixieCeilingInBytes);
 }
 
 inline void*
@@ -861,6 +885,24 @@ update(Context* c, void** p, bool* needsVisit)
   return update2(c, mask(*p), needsVisit);
 }
 
+void
+markDirty(Context* c, Fixie* f)
+{
+  if (not f->dirty) {
+    f->dirty = true;
+    f->move(&(c->dirtyTenuredFixies));
+  }
+}
+
+void
+markClean(Context* c, Fixie* f)
+{
+  if (f->dirty) {
+    f->dirty = false;
+    f->move(&(c->tenuredFixies));
+  }
+}
+
 void
 updateHeapMap(Context* c, void* p, void* target, unsigned offset, void* result)
 {
@@ -876,7 +918,7 @@ updateHeapMap(Context* c, void* p, void* target, unsigned offset, void* result)
   }
 
   if (not (c->client->isFixed(result)
-           and fixie(result)->age == FixieTenureThreshold)
+           and fixie(result)->age >= FixieTenureThreshold)
       and not seg->contains(result))
   {
     if (target and c->client->isFixed(target)) {
@@ -884,14 +926,13 @@ updateHeapMap(Context* c, void* p, void* target, unsigned offset, void* result)
       assert(c, offset == 0 or f->hasMask);
 
       if (static_cast<unsigned>(f->age + 1) >= FixieTenureThreshold) {
-        unsigned size = c->client->sizeInWords(f->body());
-
         if (DebugFixies) {
-          fprintf(stderr, "dirty fixie %p at %d\n", f, offset);
+          fprintf(stderr, "dirty fixie %p at %d (%p)\n",
+                  f, offset, f->body() + offset);
         }
 
-        f->dirty = true;
+        markDirty(c, f);
-        markBit(f->mask(size), offset);
+        markBit(f->mask(), offset);
       }
     } else if (seg->contains(p)) {
       if (Debug) {        
@@ -1227,20 +1268,23 @@ collect(Context* c, void* target, unsigned offset)
 }
 
 void
-visitDirtyFixies(Context* c)
+visitDirtyFixies(Context* c, Fixie** p)
 {
-  for (Fixie** p = &(c->dirtyFixies); *p;) {
+  while (*p) {
     Fixie* f = *p;
 
     bool wasDirty = false;
-    bool stillDirty = false;
+    bool clean = true;
-    unsigned size = c->client->sizeInWords(f->body());
+    uintptr_t* mask = f->mask();
-    uintptr_t* mask = f->mask(size);
 
     unsigned word = 0;
     unsigned bit = 0;
-    unsigned wordLimit = wordOf(size);
+    unsigned wordLimit = wordOf(f->size);
-    unsigned bitLimit = bitOf(size);
+    unsigned bitLimit = bitOf(f->size);
+
+    if (DebugFixies) {
+      fprintf(stderr, "clean fixie %p\n", f);
+    }
 
     for (; word <= wordLimit and (word < wordLimit or bit < bitLimit);
          ++ word)
@@ -1255,15 +1299,16 @@ visitDirtyFixies(Context* c)
             wasDirty = true;
 
             clearBit(mask, index);
-    
+
             if (DebugFixies) {
-              fprintf(stderr, "clean fixie %p at %d\n", f, index);
+              fprintf(stderr, "clean fixie %p at %d (%p)\n",
+                      f, index, f->body() + index);
             }
 
             collect(c, f->body(), index);
 
             if (getBit(mask, index)) {
-              stillDirty = true;
+              clean = false;
             }
           }
         }
@@ -1271,15 +1316,18 @@ visitDirtyFixies(Context* c)
       }
     }
 
+    if (DebugFixies) {
+      fprintf(stderr, "done cleaning fixie %p\n", f);
+    }
+
     assert(c, wasDirty);
 
-    if (stillDirty) {
+    if (clean) {
-      p = &(f->next);
-    } else {
-      f->dirty = false;
       *p = f->next;
-      f->move(&(c->tenuredFixies));
+      markClean(c, f);
-    } 
+    } else {
+      p = &(f->next);
+    }
   }
 }
 
@@ -1373,7 +1421,7 @@ collect2(Context* c)
   }
 
   if (c->mode == Heap::MinorCollection) {
-    visitDirtyFixies(c);
+    visitDirtyFixies(c, &(c->dirtyTenuredFixies));
   }
 
   class Visitor : public Heap::Visitor {
@@ -1469,11 +1517,11 @@ collect(Context* c, unsigned footprint)
 
     fprintf(stderr,
             " - untenured fixies:          %8d bytes\n",
-            c->untenuredFixieFootprint * BytesPerWord);
+            c->untenuredFixieFootprint);
 
     fprintf(stderr,
             " -   tenured fixies:          %8d bytes\n",
-            c->tenuredFixieFootprint * BytesPerWord);
+            c->tenuredFixieFootprint);
   }
 }
 
@@ -1487,17 +1535,25 @@ class MyHeap: public Heap {
     ::collect(&c, footprint);
   }
 
-  virtual void* allocateFixed(unsigned sizeInWords, bool objectMask,
+  virtual void* allocateFixed(Allocator* allocator, unsigned sizeInWords,
-                              unsigned* totalInBytes)
+                              bool objectMask, unsigned* totalInBytes)
   {
     *totalInBytes = Fixie::totalSize(sizeInWords, objectMask);
-    return (new (c.system->allocate(*totalInBytes))
+    return (new (allocator->allocate(*totalInBytes))
-            Fixie(sizeInWords, objectMask, &(c.fixies)))->body();
+            Fixie(sizeInWords, objectMask, &(c.fixies), false))->body();
+  }
+
+  virtual void* allocateImmortal(Allocator* allocator, unsigned sizeInWords,
+                                 bool objectMask, unsigned* totalInBytes)
+  {
+    *totalInBytes = Fixie::totalSize(sizeInWords, objectMask);
+    return (new (allocator->allocate(*totalInBytes))
+            Fixie(sizeInWords, objectMask, &(c.tenuredFixies), true))->body();
   }
 
   virtual bool needsMark(void* p) {
     if (c.client->isFixed(p)) {
-      return fixie(p)->age == FixieTenureThreshold;
+      return fixie(p)->age >= FixieTenureThreshold;
     } else {
       return c.gen2.contains(p);
     }
@@ -1507,7 +1563,7 @@ class MyHeap: public Heap {
     return target
       and not c.gen2.contains(target)
       and not (c.client->isFixed(target)
-               and fixie(target)->age == FixieTenureThreshold);
+               and fixie(target)->age >= FixieTenureThreshold);
   }
 
   virtual void mark(void* p, unsigned offset, unsigned count) {
@@ -1515,27 +1571,25 @@ class MyHeap: public Heap {
       Fixie* f = fixie(p);
       assert(&c, offset == 0 or f->hasMask);
 
-      unsigned size = c.client->sizeInWords(f->body());
+      bool dirty = false;
-
       for (unsigned i = 0; i < count; ++i) {
         void** target = static_cast<void**>(p) + offset + i;
-        if (targetNeedsMark(*target)) {
+        if (targetNeedsMark(mask(*target))) {
           if (DebugFixies) {
-            fprintf(stderr, "dirty fixie %p at %d\n", f, offset + i);
+            fprintf(stderr, "dirty fixie %p at %d (%p)\n",
+                    f, offset, f->body() + offset);
           }
 
-          f->dirty = true;
+          dirty = true;
-          markBit(f->mask(size), offset + i);
+          markBit(f->mask(), offset + i);
         }
       }
 
-      if (f->dirty) {
+      if (dirty) markDirty(&c, f);
-        f->move(&(c.dirtyFixies));
-      }
     } else {
       for (unsigned i = 0; i < count; ++i) {
         void** target = static_cast<void**>(p) + offset + i;
-        if (targetNeedsMark(*target)) {
+        if (targetNeedsMark(mask(*target))) {
           c.heapMap.set(target);
         }
       }
@@ -1598,7 +1652,7 @@ class MyHeap: public Heap {
 
   virtual void dispose() {
     c.dispose();
-    c.system->free(this);
+    c.system->free(this, sizeof(*this));
   }
 
   Context c;

src/heap.h

@@ -2,6 +2,7 @@
 #define HEAP_H
 
 #include "system.h"
+#include "zone.h"
 
 namespace vm {
 
@@ -45,8 +46,10 @@ class Heap {
 
   virtual ~Heap() { }
   virtual void collect(CollectionType type, unsigned footprint) = 0;
-  virtual void* allocateFixed(unsigned sizeInWords, bool objectMask,
+  virtual void* allocateFixed(Allocator* allocator, unsigned sizeInWords,
-                              unsigned* totalInBytes) = 0;
+                              bool objectMask, unsigned* totalInBytes) = 0;
+  virtual void* allocateImmortal(Allocator* allocator, unsigned sizeInWords,
+                                 bool objectMask, unsigned* totalInBytes) = 0;
   virtual bool needsMark(void* p) = 0;
   virtual void mark(void* p, unsigned offset, unsigned count) = 0;
   virtual void pad(void* p, unsigned extra) = 0;

src/interpret.cpp

@@ -3015,8 +3015,8 @@ class MyProcessor: public Processor {
     }
   }
 
-  virtual void dispose(vm::Thread*) {
+  virtual void dispose(vm::Thread* t) {
-    // ignore
+    s->free(t, sizeof(Thread));
   }
 
   virtual void dispose() {

src/jnienv.cpp

@@ -96,9 +96,9 @@ GetStringUTFChars(Thread* t, jstring s, jboolean* isCopy)
 }
 
 void JNICALL
-ReleaseStringUTFChars(Thread* t, jstring, const char* chars)
+ReleaseStringUTFChars(Thread* t, jstring s, const char* chars)
 {
-  t->m->system->free(chars);
+  t->m->system->free(chars, stringLength(t, *s) + 1);
 }
 
 jsize JNICALL
@@ -1393,119 +1393,126 @@ void JNICALL
 ReleaseBooleanArrayElements(Thread* t, jbooleanArray array, jboolean* p,
                             jint mode)
 {
+  unsigned size = booleanArrayLength(t, *array) * sizeof(jboolean);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = booleanArrayLength(t, *array) * sizeof(jboolean);
     if (size) {
       memcpy(&booleanArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 
 void JNICALL
 ReleaseByteArrayElements(Thread* t, jbyteArray array, jbyte* p, jint mode)
 {
+  unsigned size = byteArrayLength(t, *array) * sizeof(jbyte);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = byteArrayLength(t, *array) * sizeof(jbyte);
     if (size) {
       memcpy(&byteArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 
 void JNICALL
 ReleaseCharArrayElements(Thread* t, jcharArray array, jchar* p, jint mode)
 {
+  unsigned size = charArrayLength(t, *array) * sizeof(jchar);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = charArrayLength(t, *array) * sizeof(jchar);
     if (size) {
       memcpy(&charArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 
 void JNICALL
 ReleaseShortArrayElements(Thread* t, jshortArray array, jshort* p, jint mode)
 {
+  unsigned size = shortArrayLength(t, *array) * sizeof(jshort);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = shortArrayLength(t, *array) * sizeof(jshort);
     if (size) {
       memcpy(&shortArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 
 void JNICALL
 ReleaseIntArrayElements(Thread* t, jintArray array, jint* p, jint mode)
 {
+  unsigned size = intArrayLength(t, *array) * sizeof(jint);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = intArrayLength(t, *array) * sizeof(jint);
     if (size) {
       memcpy(&intArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 
 void JNICALL
 ReleaseLongArrayElements(Thread* t, jlongArray array, jlong* p, jint mode)
 {
+  unsigned size = longArrayLength(t, *array) * sizeof(jlong);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = longArrayLength(t, *array) * sizeof(jlong);
     if (size) {
       memcpy(&longArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 
 void JNICALL
 ReleaseFloatArrayElements(Thread* t, jfloatArray array, jfloat* p, jint mode)
 {
+  unsigned size = floatArrayLength(t, *array) * sizeof(jfloat);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = floatArrayLength(t, *array) * sizeof(jfloat);
     if (size) {
       memcpy(&floatArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 
@@ -1513,17 +1520,18 @@ void JNICALL
 ReleaseDoubleArrayElements(Thread* t, jdoubleArray array, jdouble* p,
                            jint mode)
 {
+  unsigned size = doubleArrayLength(t, *array) * sizeof(jdouble);
+
   if (mode == 0 or mode == JNI_COMMIT) {
     ENTER(t, Thread::ActiveState);
     
-    unsigned size = doubleArrayLength(t, *array) * sizeof(jdouble);
     if (size) {
       memcpy(&doubleArrayBody(t, *array, 0), p, size);
     }
   }
 
   if (mode == 0 or mode == JNI_ABORT) {
-    t->m->system->free(p);
+    t->m->system->free(p, size);
   }
 }
 

src/machine.cpp

@@ -482,7 +482,7 @@ void
 postCollect(Thread* t)
 {
 #ifdef VM_STRESS
-  t->m->system->free(t->defaultHeap);
+  t->m->system->free(t->defaultHeap, Thread::HeapSizeInBytes);
   t->defaultHeap = static_cast<uintptr_t*>
     (t->m->system->allocate(Thread::HeapSizeInBytes));
 #endif
@@ -813,7 +813,7 @@ parsePool(Thread* t, Stream& s)
       i += parsePoolEntry(t, s, index, pool, i);
     }
 
-    t->m->system->free(index);
+    t->m->system->free(index, count * 4);
 
     s.setPosition(end);
   }
@@ -1758,7 +1758,7 @@ class HeapClient: public Heap::Client {
   }
 
   virtual void dispose() {
-    m->system->free(this);
+    m->system->free(this, sizeof(*this));
   }
     
  private:
@@ -1832,16 +1832,16 @@ Machine::dispose()
   for (Reference* r = jniReferences; r;) {
     Reference* t = r;
     r = r->next;
-    system->free(t);
+    system->free(t, sizeof(*t));
   }
 
   for (unsigned i = 0; i < heapPoolIndex; ++i) {
-    system->free(heapPool[i]);
+    system->free(heapPool[i], Thread::HeapSizeInBytes);
   }
 
   heap->dispose();
 
-  system->free(this);
+  system->free(this, sizeof(*this));
 }
 
 Thread::Thread(Machine* m, object javaThread, Thread* parent):
@@ -1992,15 +1992,13 @@ Thread::exit()
 void
 Thread::dispose()
 {
-  m->processor->dispose(this);
-
   if (systemThread) {
     systemThread->dispose();
   }
 
-  m->system->free(defaultHeap);
+  m->system->free(defaultHeap, Thread::HeapSizeInBytes);
 
-  m->system->free(this);
+  m->processor->dispose(this);
 }
 
 void
@@ -2144,7 +2142,18 @@ enter(Thread* t, Thread::State s)
 }
 
 object
-allocate2(Thread* t, unsigned sizeInBytes, bool objectMask, bool fixed)
+allocate2(Thread* t, unsigned sizeInBytes, bool objectMask)
+{
+  return allocate3
+    (t, t->m->system,
+     ceiling(sizeInBytes, BytesPerWord) > Thread::HeapSizeInWords ?
+     Machine::FixedAllocation : Machine::MovableAllocation,
+     sizeInBytes, objectMask);
+}
+
+object
+allocate3(Thread* t, Allocator* allocator, Machine::AllocationType type,
+          unsigned sizeInBytes, bool objectMask)
 {
   ACQUIRE_RAW(t, t->m->stateLock);
 
@@ -2154,7 +2163,7 @@ allocate2(Thread* t, unsigned sizeInBytes, bool objectMask, bool fixed)
     ENTER(t, Thread::IdleState);
   }
 
-  if (fixed) {
+  if (type == Machine::FixedAllocation) {
     if (t->m->fixedFootprint + sizeInBytes
         > Machine::FixedFootprintThresholdInBytes)
     {
@@ -2179,20 +2188,37 @@ allocate2(Thread* t, unsigned sizeInBytes, bool objectMask, bool fixed)
     ENTER(t, Thread::ExclusiveState);
     collect(t, Heap::MinorCollection);
   }
+  
+  switch (type) {
+  case Machine::MovableAllocation: {
+    return allocateSmall(t, sizeInBytes);
+  }
 
-  if (fixed) {
+  case Machine::FixedAllocation: {
     unsigned total;
     object o = static_cast<object>
       (t->m->heap->allocateFixed
-       (ceiling(sizeInBytes, BytesPerWord), objectMask, &total));
+       (allocator, ceiling(sizeInBytes, BytesPerWord), objectMask, &total));
 
     cast<uintptr_t>(o, 0) = FixedMark;
 
     t->m->fixedFootprint += total;
 
     return o;
-  } else {
+  }
-    return allocateSmall(t, sizeInBytes);
+
+  case Machine::ImmortalAllocation: {
+    unsigned total;
+    object o = static_cast<object>
+      (t->m->heap->allocateImmortal
+       (allocator, ceiling(sizeInBytes, BytesPerWord), objectMask, &total));
+
+    cast<uintptr_t>(o, 0) = FixedMark;
+
+    return o;
+  }
+
+  default: abort(t);
   }
 }
 
@@ -2800,7 +2826,7 @@ collect(Thread* t, Heap::CollectionType type)
   killZombies(t, m->rootThread);
 
   for (unsigned i = 0; i < m->heapPoolIndex; ++i) {
-    m->system->free(m->heapPool[i]);
+    m->system->free(m->heapPool[i], Thread::HeapSizeInBytes);
   }
   m->heapPoolIndex = 0;
 

src/machine.h

@@ -1105,10 +1105,10 @@ class Machine {
 #include "type-enums.cpp"
   };
 
-  enum State {
+  enum AllocationType {
-    UnsafeState,
+    MovableAllocation,
-    BootState,
+    FixedAllocation,
-    ActiveState
+    ImmortalAllocation
   };
 
   Machine(System* system, Finder* finder, Processor* processor);
@@ -1306,7 +1306,7 @@ dispose(Thread* t, Reference* r)
   if (r->next) {
     r->next->handle = r->handle;
   }
-  t->m->system->free(r);
+  t->m->system->free(r, sizeof(*r));
 }
 
 void
@@ -1411,7 +1411,11 @@ expect(Thread* t, bool v)
 }
 
 object
-allocate2(Thread* t, unsigned sizeInBytes, bool objectMask, bool fixed);
+allocate2(Thread* t, unsigned sizeInBytes, bool objectMask);
+
+object
+allocate3(Thread* t, Allocator* allocator, Machine::AllocationType type,
+          unsigned sizeInBytes, bool objectMask);
 
 inline object
 allocateSmall(Thread* t, unsigned sizeInBytes)
@@ -1431,8 +1435,7 @@ allocate(Thread* t, unsigned sizeInBytes, bool objectMask)
                >= Thread::HeapSizeInWords
                or t->m->exclusive))
   {
-    return allocate2(t, sizeInBytes, objectMask,
+    return allocate2(t, sizeInBytes, objectMask);
-                     sizeInBytes > Thread::HeapSizeInBytes);
   } else {
     return allocateSmall(t, sizeInBytes);
   }
@@ -1461,12 +1464,12 @@ set(Thread* t, object target, unsigned offset, object value)
 }
 
 inline void
-setObjectClass(Thread* t, object o, object value)
+setObjectClass(Thread*, object o, object value)
 {
-  set(t, o, 0,
+  cast<object>(o, 0)
-      reinterpret_cast<object>
+    = reinterpret_cast<object>
-      (reinterpret_cast<uintptr_t>(value)
+    (reinterpret_cast<uintptr_t>(value)
-       | reinterpret_cast<uintptr_t>(cast<object>(o, 0)) & (~PointerMask)));
+     | reinterpret_cast<uintptr_t>(cast<object>(o, 0)) & (~PointerMask));
 }
 
 object&

src/posix.cpp

@@ -115,6 +115,45 @@ const unsigned Notified = 1 << 1;
 
 class MySystem: public System {
  public:
+  class CodeAllocator: public Allocator {
+   public:
+    CodeAllocator(MySystem* s): s(s) { }
+
+    virtual void* tryAllocate(unsigned size) {
+      assert(s, size % LikelyPageSizeInBytes == 0);
+
+#ifdef __x86_64__
+      void* p = mmap(0, size, PROT_EXEC | PROT_READ | PROT_WRITE,
+                     MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT, -1, 0);
+
+      if (p == MAP_FAILED) {
+        return 0;
+      } else {
+        return p;
+      }
+#else
+      return s->tryAllocate(size);
+#endif
+    }
+
+    virtual void* allocate(unsigned size) {
+      void* p = tryAllocate(size);
+      expect(s, p);
+      return p;
+    }
+
+    virtual void free(const void* p, unsigned size) {
+#ifdef __x86_64__
+      int r UNUSED = munmap(const_cast<void*>(p), size);
+      assert(s, r == 0);
+#else
+      s->free(p, size);
+#endif
+    }
+
+    MySystem* s;
+  };
+
   class Thread: public System::Thread {
    public:
     Thread(System* s, System::Runnable* r):
@@ -143,7 +182,7 @@ class MySystem: public System {
     }
 
     virtual void dispose() {
-      s->free(this);
+      s->free(this, sizeof(*this));
     }
 
     pthread_t thread;
@@ -338,7 +377,7 @@ class MySystem: public System {
     virtual void dispose() {
       expect(s, owner_ == 0);
       pthread_mutex_destroy(&mutex);
-      s->free(this);
+      s->free(this, sizeof(*this));
     }
 
     System* s;
@@ -369,7 +408,7 @@ class MySystem: public System {
       int r UNUSED = pthread_key_delete(key);
       expect(s, r == 0);
 
-      s->free(this);
+      s->free(this, sizeof(*this));
     }
 
     System* s;
@@ -378,8 +417,8 @@ class MySystem: public System {
 
   class Region: public System::Region {
    public:
-    Region(System* system, uint8_t* start, size_t length):
+    Region(System* s, uint8_t* start, size_t length):
-      system(system),
+      s(s),
       start_(start),
       length_(length)
     { }
@@ -396,21 +435,22 @@ class MySystem: public System {
       if (start_) {
         munmap(start_, length_);
       }
-      system->free(this);
+      s->free(this, sizeof(*this));
     }
 
-    System* system;
+    System* s;
     uint8_t* start_;
     size_t length_;
   };
 
   class Library: public System::Library {
    public:
-    Library(System* s, void* p, const char* name, bool mapName,
+    Library(System* s, void* p, const char* name, unsigned nameLength,
-            System::Library* next):
+            bool mapName, System::Library* next):
       s(s),
       p(p),
       name_(name),
+      nameLength(nameLength),
       mapName_(mapName),
       next_(next)
     { }
@@ -443,20 +483,21 @@ class MySystem: public System {
       }
 
       if (name_) {
-        s->free(name_);
+        s->free(name_, nameLength + 1);
       }
 
-      s->free(this);
+      s->free(this, sizeof(*this));
     }
 
     System* s;
     void* p;
     const char* name_;
+    unsigned nameLength;
     bool mapName_;
     System::Library* next_;
   };
 
-  MySystem(unsigned limit): limit(limit), count(0) {
+  MySystem(unsigned limit): limit(limit), count(0), codeAllocator_(this) {
     pthread_mutex_init(&mutex, 0);
 
     struct sigaction sa;
@@ -469,10 +510,6 @@ class MySystem: public System {
     expect(this, rv == 0);
   }
 
-  virtual bool success(Status s) {
-    return s == 0;
-  }
-
   virtual void* tryAllocate(unsigned size) {
     ACQUIRE(mutex);
 
@@ -484,37 +521,65 @@ class MySystem: public System {
     if (count + size > limit) {
       return 0;
     } else {
+#ifndef NDEBUG
       uintptr_t* up = static_cast<uintptr_t*>
         (malloc(size + sizeof(uintptr_t)));
       if (up == 0) {
-        sysAbort(this);
+        return 0;
       } else {
         *up = size;
         count += *up;
+      
         return up + 1;
       }
+#else
+      void* p = malloc(size);
+      if (p == 0) {
+        return 0;
+      } else {
+        count += size;
+        return p;
+      }
+#endif
     }
   }
 
-  virtual void free(const void* p) {
+  virtual void free(const void* p, unsigned size) {
     ACQUIRE(mutex);
 
+    if (Verbose) {
+      fprintf(stderr, "free %d; count: %d; limit: %d\n",
+              size, count, limit);
+    }
+
     if (p) {
+#ifndef NDEBUG
       const uintptr_t* up = static_cast<const uintptr_t*>(p) - 1;
-      if (count < *up) {
+
-        abort();
+      if (*up != size) abort();
-      }
+      if (count < *up) abort();
+
       count -= *up;
 
-      if (Verbose) {
-        fprintf(stderr, "free %"ULD"; count: %d; limit: %d\n",
-                *up, count, limit);
-      }
-
       ::free(const_cast<uintptr_t*>(up));
+#else
+      if (count < size) abort();
+
+      count -= size;
+
+      ::free(const_cast<void*>(p));
+#endif
     }
   }
 
+  virtual Allocator* codeAllocator() {
+    return &codeAllocator_;
+  }
+
+  virtual bool success(Status s) {
+    return s == 0;
+  }
+
   virtual Status attach(Runnable* r) {
     Thread* t = new (System::allocate(sizeof(Thread))) Thread(this, r);
     t->thread = pthread_self();
@@ -632,7 +697,7 @@ class MySystem: public System {
       }
 
       *lib = new (System::allocate(sizeof(Library)))
-        Library(this, p, n, mapName, next);
+        Library(this, p, n, nameLength, mapName, next);
       return 0;
     } else {
 //       fprintf(stderr, "dlerror: %s\n", dlerror());
@@ -668,6 +733,7 @@ class MySystem: public System {
   pthread_mutex_t mutex;
   unsigned limit;
   unsigned count;
+  CodeAllocator codeAllocator_;
 };
 
 } // namespace

src/system.h

@@ -2,17 +2,10 @@
 #define SYSTEM_H
 
 #include "common.h"
+#include "allocator.h"
 
 namespace vm {
 
-class Allocator {
- public:
-  virtual ~Allocator() { }
-  virtual void* tryAllocate(unsigned size) = 0;
-  virtual void* allocate(unsigned size) = 0;
-  virtual void free(const void*) = 0;
-};
-
 class System: public Allocator {
  public:
   typedef intptr_t Status;
@@ -90,6 +83,7 @@ class System: public Allocator {
 
   virtual ~System() { }
 
+  virtual Allocator* codeAllocator() = 0;
   virtual bool success(Status) = 0;
   virtual Status attach(Runnable*) = 0;
   virtual Status start(Runnable*) = 0;

src/vector.h

@@ -21,7 +21,7 @@ class Vector {
 
   void dispose() {
     if (data and minimumCapacity >= 0) {
-      s->free(data);
+      s->free(data, capacity);
     }
   }
 
@@ -43,7 +43,7 @@ class Vector {
       uint8_t* newData = static_cast<uint8_t*>(s->allocate(newCapacity));
       if (data) {
         memcpy(newData, data, position);
-        s->free(data);
+        s->free(data, capacity);
       }
       data = newData;
       capacity = newCapacity;

src/windows.cpp

@@ -444,10 +444,6 @@ class MySystem: public System {
     assert(this, mutex);
   }
 
-  virtual bool success(Status s) {
-    return s == 0;
-  }
-
   virtual void* tryAllocate(unsigned size) {
     ACQUIRE(this, mutex);
 
@@ -459,38 +455,65 @@ class MySystem: public System {
     if (count + size > limit) {
       return 0;
     } else {
+#ifndef NDEBUG
       uintptr_t* up = static_cast<uintptr_t*>
         (malloc(size + sizeof(uintptr_t)));
       if (up == 0) {
-        sysAbort(this);
+        return 0;
       } else {
         *up = size;
         count += *up;
       
         return up + 1;
       }
+#else
+      void* p = malloc(size);
+      if (p == 0) {
+        return 0;
+      } else {
+        count += size;
+        return p;
+      }
+#endif
     }
   }
 
-  virtual void free(const void* p) {
+  virtual void free(const void* p, unsigned size) {
     ACQUIRE(this, mutex);
 
+    if (Verbose) {
+      fprintf(stderr, "free %d; count: %d; limit: %d\n",
+              size, count, limit);
+    }
+
     if (p) {
+#ifndef NDEBUG
       const uintptr_t* up = static_cast<const uintptr_t*>(p) - 1;
-      if (count < *up) {
+
-        abort();
+      if (*up != size) abort();
-      }
+      if (count < *up) abort();
+
       count -= *up;
 
-      if (Verbose) {
-        fprintf(stderr, "free %"ULD"; count: %d; limit: %d\n",
-                *up, count, limit);
-      }
-
       ::free(const_cast<uintptr_t*>(up));
+#else
+      if (count < size) abort();
+
+      count -= size;
+
+      ::free(const_cast<void*>(p));
+#endif
     }
   }
 
+  virtual Allocator* codeAllocator() {
+    return this;
+  }
+
+  virtual bool success(Status s) {
+    return s == 0;
+  }
+
   virtual Status attach(Runnable* r) {
     Thread* t = new (System::allocate(sizeof(Thread))) Thread(this, r);
     bool success UNUSED = DuplicateHandle

src/zone.h

@@ -2,25 +2,28 @@
 #define ZONE_H
 
 #include "system.h"
+#include "allocator.h"
 
 namespace vm {
 
-class Zone {
+class Zone: public Allocator {
  public:
   class Segment {
    public:
-    Segment(Segment* next): next(next) { }
+    Segment(Segment* next, unsigned size): next(next), size(size) { }
 
     Segment* next;
+    uintptr_t size;
     uint8_t data[0];
   };
 
-  Zone(System* s, unsigned minimumCapacity):
+  Zone(System* s, Allocator* a, unsigned minimumFootprint):
     s(s),
+    a(a),
     segment(0),
     position(0),
-    capacity(0),
+    minimumFootprint(minimumFootprint < sizeof(Segment) ? 0 :
-    minimumCapacity(minimumCapacity)
+                     minimumFootprint - sizeof(Segment))
   { }
 
   ~Zone() {
@@ -30,31 +33,63 @@ class Zone {
   void dispose() {
     for (Segment* seg = segment, *next; seg; seg = next) {
       next = seg->next;
-      s->free(seg);
+      a->free(seg, sizeof(Segment) + seg->size);
     }
   }
 
-  void ensure(unsigned space) {
+  bool ensure(unsigned space) {
-    if (position + space > capacity) {
+    if (segment == 0 or position + space > segment->size) {
-      capacity = max(space, max(minimumCapacity, capacity * 2));
+      unsigned size = max
-      segment = new (s->allocate(sizeof(Segment) + capacity)) Segment(segment);
+        (space, max
+         (minimumFootprint, segment == 0 ? 0 : segment->size * 2))
+        + sizeof(Segment);
+
+      // pad to page size
+      size = (size + (LikelyPageSizeInBytes - 1))
+        & ~(LikelyPageSizeInBytes - 1);
+
+      void* p = a->tryAllocate(size);
+      if (p == 0) {
+        size = space + sizeof(Segment);
+        void* p = a->tryAllocate(size);
+        if (p == 0) {
+          return false;
+        }
+      }
+
+      segment = new (p) Segment(segment, size - sizeof(Segment));
       position = 0;
     }
+    return true;
   }
 
-  void* allocate(unsigned size) {
+  virtual void* tryAllocate(unsigned size) {
     size = pad(size);
-    ensure(size);
+    if (ensure(size)) {
-    void* r = segment->data + position;
+      void* r = segment->data + position;
-    position += size;
+      position += size;
-    return r;
+      return r;
+    } else {
+      return 0;
+    }
+  }
+
+  virtual void* allocate(unsigned size) {
+    void* p = tryAllocate(size);
+    expect(s, p);
+    return p;
+  }
+
+  virtual void free(const void*, unsigned) {
+    // not supported
+    abort(s);
   }
   
   System* s;
+  Allocator* a;
   Segment* segment;
   unsigned position;
-  unsigned capacity;
+  unsigned minimumFootprint;
-  unsigned minimumCapacity;
 };
 
 } // namespace vm