diff --git a/makefile b/makefile index a3a6519c2b..e8645506f4 100755 --- a/makefile +++ b/makefile @@ -165,12 +165,13 @@ rdynamic = -rdynamic warnings = -Wall -Wextra -Werror -Wunused-parameter -Winit-self \ -Wno-non-virtual-dtor +target-cflags = -DTARGET_BYTES_PER_WORD=$(pointer-size) + common-cflags = $(warnings) -fno-rtti -fno-exceptions \ "-I$(JAVA_HOME)/include" -idirafter $(src) -I$(build) $(classpath-cflags) \ -D__STDC_LIMIT_MACROS -D_JNI_IMPLEMENTATION_ -DAVIAN_VERSION=\"$(version)\" \ -DUSE_ATOMIC_OPERATIONS -DAVIAN_JAVA_HOME=\"$(javahome)\" \ - -DAVIAN_EMBED_PREFIX=\"$(embed-prefix)\" \ - -DTARGET_BYTES_PER_WORD=$(pointer-size) + -DAVIAN_EMBED_PREFIX=\"$(embed-prefix)\" $(target-cflags) ifneq (,$(filter i386 x86_64,$(arch))) ifeq ($(use-frame-pointer),true) @@ -209,9 +210,11 @@ shared = -shared openjdk-extra-cflags = -fvisibility=hidden +bootimage-cflags = -DTARGET_BYTES_PER_WORD=$(pointer-size) + ifeq ($(build-arch),powerpc) ifneq ($(arch),$(build-arch)) - cflags += -DTARGET_OPPOSITE_ENDIAN + bootimage-cflags += -DTARGET_OPPOSITE_ENDIAN endif endif @@ -223,7 +226,7 @@ ifeq ($(arch),powerpc) pointer-size = 4 ifneq ($(arch),$(build-arch)) - cflags += -DTARGET_OPPOSITE_ENDIAN + bootimage-cflags += -DTARGET_OPPOSITE_ENDIAN endif ifneq ($(platform),darwin) @@ -264,7 +267,7 @@ endif ifeq ($(platform),linux) - cflags += -DTARGET_PLATFORM_LINUX + bootimage-cflags += -DTARGET_PLATFORM_LINUX endif ifeq ($(build-platform),darwin) @@ -274,7 +277,7 @@ ifeq ($(build-platform),darwin) endif ifeq ($(platform),darwin) - cflags += -DTARGET_PLATFORM_DARWIN + bootimage-cflags += -DTARGET_PLATFORM_DARWIN ifeq (${OSX_SDK_SYSROOT},) OSX_SDK_SYSROOT = 10.4u @@ -350,7 +353,7 @@ ifeq ($(platform),darwin) endif ifeq ($(platform),windows) - cflags += -DTARGET_PLATFORM_WINDOWS + bootimage-cflags += -DTARGET_PLATFORM_WINDOWS inc = "$(root)/win32/include" lib = "$(root)/win32/lib" @@ -520,10 +523,12 @@ vm-sources = \ vm-asm-sources = $(src)/$(asm).S +target-asm = $(asm) + ifeq ($(process),compile) vm-sources += \ $(src)/compiler.cpp \ - $(src)/$(asm).cpp + $(src)/$(target-asm).cpp vm-asm-sources += $(src)/compile-$(asm).S endif @@ -902,6 +907,8 @@ $(bootimage-generator): openjdk-src=$(openjdk-src) \ bootimage-generator= \ build-bootimage-generator=$(bootimage-generator) \ + target-cflags="$(bootimage-cflags)" \ + target-asm=$(asm) \ $(bootimage-generator) $(build-bootimage-generator): \ diff --git a/src/bootimage-template.cpp b/src/bootimage-template.cpp new file mode 100644 index 0000000000..d96a586fcd --- /dev/null +++ b/src/bootimage-template.cpp @@ -0,0 +1,30 @@ +const unsigned NAME(BootMask) = (~static_cast(0)) + / NAME(BytesPerWord); + +const unsigned NAME(BootShift) = 32 - log(NAME(BytesPerWord)); + +const unsigned NAME(BootFlatConstant) = 1 << NAME(BootShift); +const unsigned NAME(BootHeapOffset) = 1 << (NAME(BootShift) + 1); + +inline unsigned +LABEL(codeMapSize)(unsigned codeSize) +{ + return ceiling(codeSize, TargetBitsPerWord) * TargetBytesPerWord; +} + +inline unsigned +LABEL(heapMapSize)(unsigned heapSize) +{ + return ceiling(heapSize, TargetBitsPerWord * TargetBytesPerWord) + * TargetBytesPerWord; +} + +inline object +LABEL(bootObject)(LABEL(uintptr_t)* heap, unsigned offset) +{ + if (offset) { + return reinterpret_cast(heap + offset - 1); + } else { + return 0; + } +} diff --git a/src/bootimage.cpp b/src/bootimage.cpp index 0626eb5c77..99d7a40086 100644 --- a/src/bootimage.cpp +++ b/src/bootimage.cpp @@ -8,7 +8,6 @@ There is NO WARRANTY for this software. See license.txt for details. */ -#include "bootimage.h" #include "heap.h" #include "heapwalk.h" #include "common.h" @@ -40,6 +39,7 @@ const bool DebugNativeTarget = false; enum Type { Type_none, Type_object, + Type_object_nogc, Type_int8_t, Type_uint8_t, Type_int16_t, @@ -60,13 +60,19 @@ enum Type { class Field { public: - Field(Type type, unsigned offset, unsigned targetOffset): - type(type), offset(offset), targetOffset(targetOffset) + Field() { } + + Field(Type type, unsigned buildOffset, unsigned buildSize, + unsigned targetOffset, unsigned targetSize): + type(type), buildOffset(buildOffset), buildSize(buildSize), + targetOffset(targetOffset), targetSize(targetSize) { } Type type; - unsigned offset; + unsigned buildOffset; + unsigned buildSize; unsigned targetOffset; + unsigned targetSize; }; class TypeMap { @@ -155,6 +161,108 @@ endsWith(const char* suffix, const char* s, unsigned length) and memcmp(suffix, s + (length - suffixLength), suffixLength) == 0; } +object +getNonStaticFields(Thread* t, object typeMaps, object c, object fields, + unsigned* count, object* array) +{ + PROTECT(t, typeMaps); + PROTECT(t, c); + PROTECT(t, fields); + + *array = hashMapFind(t, typeMaps, c, objectHash, objectEqual); + + if (*array) { + *count += reinterpret_cast(&byteArrayBody(t, *array, 0)) + ->fixedFieldCount; + } else { + if (classSuper(t, c)) { + fields = getNonStaticFields + (t, typeMaps, classSuper(t, c), fields, count, array); + } + + if (classFieldTable(t, c)) { + for (unsigned i = 0; i < arrayLength(t, classFieldTable(t, c)); ++i) { + object field = arrayBody(t, classFieldTable(t, c), i); + + if ((fieldFlags(t, field) & ACC_STATIC) == 0) { + ++ (*count); + fields = vectorAppend(t, fields, field); + } + } + } + } + + return vectorAppend(t, fields, 0); +} + +object +allFields(Thread* t, object typeMaps, object c, unsigned* count, object* array) +{ + PROTECT(t, typeMaps); + PROTECT(t, c); + + object fields = makeVector(t, 0, 0); + PROTECT(t, fields); + + *array = hashMapFind(t, typeMaps, c, objectHash, objectEqual); + + bool includeMembers; + if (*array) { + includeMembers = false; + *count += reinterpret_cast(&byteArrayBody(t, *array, 0)) + ->fixedFieldCount; + } else if (classSuper(t, c)) { + includeMembers = true; + fields = getNonStaticFields + (t, typeMaps, classSuper(t, c), fields, count, array); + } + + if (classFieldTable(t, c)) { + for (unsigned i = 0; i < arrayLength(t, classFieldTable(t, c)); ++i) { + object field = arrayBody(t, classFieldTable(t, c), i); + + if (includeMembers or (fieldFlags(t, field) & ACC_STATIC)) { + ++ (*count); + fields = vectorAppend(t, fields, field); + } + } + } + + return fields; +} + +TypeMap* +classTypeMap(Thread* t, object typeMaps, object p) +{ + return reinterpret_cast + (&byteArrayBody + (t, hashMapFind(t, typeMaps, p, objectHash, objectEqual), 0)); +} + +TypeMap* +typeMap(Thread* t, object typeMaps, object p) +{ + return reinterpret_cast + (&byteArrayBody + (t, objectClass(t, p) == type(t, Machine::SingletonType) + ? hashMapFind(t, typeMaps, p, objectHash, objectEqual) + : hashMapFind(t, typeMaps, objectClass(t, p), objectHash, objectEqual), + 0)); +} + +unsigned +targetFieldOffset(Thread* t, object typeMaps, object field) +{ + // if (strcmp(reinterpret_cast + // (&byteArrayBody(t, className(t, fieldClass(t, field)), 0)), + // "java/lang/Throwable") == 0) trap(); + + return ((fieldFlags(t, field) & ACC_STATIC) + ? typeMap(t, typeMaps, classStaticTable(t, fieldClass(t, field))) + : classTypeMap(t, typeMaps, fieldClass(t, field))) + ->targetFixedOffsets()[fieldOffset(t, field)]; +} + object makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, uintptr_t* codeMap, const char* className, @@ -168,8 +276,22 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, object calls = 0; PROTECT(t, calls); + object methods = 0; + PROTECT(t, methods); + DelayedPromise* addresses = 0; + class MyOffsetResolver: public OffsetResolver { + public: + MyOffsetResolver(object* typeMaps): typeMaps(typeMaps) { } + + virtual unsigned fieldOffset(Thread* t, object field) { + return targetFieldOffset(t, *typeMaps, field); + } + + object* typeMaps; + } resolver(&typeMaps); + Finder* finder = static_cast (systemClassLoaderFinder(t, root(t, Machine::BootLoader))); @@ -272,7 +394,8 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, = i * TargetBytesPerWord; new (map->fixedFields() + i) Field - (types[i], i * BytesPerWord, i * TargetBytesPerWord); + (types[i], i * BytesPerWord, BytesPerWord, + i * TargetBytesPerWord, TargetBytesPerWord); } hashMapInsert @@ -281,101 +404,135 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, objectHash); } } + + // if (strcmp(name, "java/lang/System$Property.class") == 0) trap(); - if (classFieldTable(t, c)) { - unsigned count = arrayLength(t, classFieldTable(t, c)); + { object array = 0; + PROTECT(t, array); - Type memberTypes[count + 1]; - memberTypes[0] = Type_object; - unsigned buildMemberOffsets[count + 1]; - buildMemberOffsets[0] = 0; - unsigned targetMemberOffsets[count + 1]; - targetMemberOffsets[0] = 0; - unsigned memberIndex = 1; - unsigned buildMemberOffset = BytesPerWord; - unsigned targetMemberOffset = TargetBytesPerWord; + unsigned count = 0; + object fields = allFields(t, typeMaps, c, &count, &array); + PROTECT(t, fields); + + Field memberFields[count + 1]; + + unsigned memberIndex; + unsigned buildMemberOffset; + unsigned targetMemberOffset; + + if (array) { + memberIndex = 0; + buildMemberOffset = 0; + targetMemberOffset = 0; + + TypeMap* map = reinterpret_cast + (&byteArrayBody(t, array, 0)); + + for (unsigned j = 0; j < map->fixedFieldCount; ++j) { + Field* f = map->fixedFields() + j; + + memberFields[memberIndex] = *f; + + targetMemberOffset += f->targetSize; + + ++ memberIndex; + } + } else { + new (memberFields) Field + (Type_object, 0, BytesPerWord, 0, TargetBytesPerWord); + + memberIndex = 1; + buildMemberOffset = BytesPerWord; + targetMemberOffset = TargetBytesPerWord; + } + + Field staticFields[count + 2]; + + new (staticFields) Field + (Type_object, 0, BytesPerWord, 0, TargetBytesPerWord); + + new (staticFields + 1) Field + (Type_intptr_t, BytesPerWord, BytesPerWord, TargetBytesPerWord, + TargetBytesPerWord); - Type staticTypes[count + 2]; - staticTypes[0] = Type_object; - staticTypes[1] = Type_intptr_t; - unsigned buildStaticOffsets[count + 2]; - buildStaticOffsets[0] = 0; - buildStaticOffsets[1] = BytesPerWord; - unsigned targetStaticOffsets[count + 2]; - targetStaticOffsets[0] = 0; - targetStaticOffsets[1] = TargetBytesPerWord; unsigned staticIndex = 2; unsigned buildStaticOffset = BytesPerWord * 2; unsigned targetStaticOffset = TargetBytesPerWord * 2; - for (unsigned i = 0; i < count; ++i) { - object field = arrayBody(t, classFieldTable(t, c), i); - unsigned size = fieldSize(t, fieldCode(t, field)); + for (unsigned i = 0; i < vectorSize(t, fields); ++i) { + object field = vectorBody(t, fields, i); + if (field) { + unsigned buildSize = fieldSize(t, fieldCode(t, field)); + unsigned targetSize = buildSize; - Type type; - switch (fieldCode(t, field)) { - case ObjectField: - type = Type_object; - size = TargetBytesPerWord; - break; + Type type; + switch (fieldCode(t, field)) { + case ObjectField: + type = Type_object; + targetSize = TargetBytesPerWord; + break; - case ByteField: - case BooleanField: - type = Type_int8_t; - break; + case ByteField: + case BooleanField: + type = Type_int8_t; + break; - case CharField: - case ShortField: - type = Type_int8_t; - break; + case CharField: + case ShortField: + type = Type_int8_t; + break; - case FloatField: - case IntField: - type = Type_int32_t; - break; + case FloatField: + case IntField: + type = Type_int32_t; + break; - case LongField: - case DoubleField: - type = Type_int64_t; - break; + case LongField: + case DoubleField: + type = Type_int64_t; + break; - default: abort(t); - } - - if (fieldFlags(t, field) & ACC_STATIC) { - staticTypes[staticIndex] = type; - - while (targetStaticOffset % size) { - ++ targetStaticOffset; + default: abort(t); } - targetStaticOffsets[staticIndex] = targetStaticOffset; + if (fieldFlags(t, field) & ACC_STATIC) { + while (targetStaticOffset % targetSize) { + ++ targetStaticOffset; + } - targetStaticOffset += size; + buildStaticOffset = fieldOffset(t, field); - buildStaticOffset = fieldOffset(t, field); - buildStaticOffsets[staticIndex] = buildStaticOffset; + new (staticFields + staticIndex) Field + (type, buildStaticOffset, buildSize, targetStaticOffset, + targetSize); - ++ staticIndex; + targetStaticOffset += targetSize; + + ++ staticIndex; + } else { + while (targetMemberOffset % targetSize) { + ++ targetMemberOffset; + } + + buildMemberOffset = fieldOffset(t, field); + + new (memberFields + memberIndex) Field + (type, buildMemberOffset, buildSize, targetMemberOffset, + targetSize); + + targetMemberOffset += targetSize; + + ++ memberIndex; + } } else { - memberTypes[memberIndex] = type; - - while (targetMemberOffset % size) { - ++ targetMemberOffset; - } - - targetMemberOffsets[memberIndex] = targetMemberOffset; - - targetMemberOffset += size; - - buildMemberOffset = fieldOffset(t, field); - buildMemberOffsets[memberIndex] = buildMemberOffset; - - ++ memberIndex; + targetMemberOffset = pad(targetMemberOffset, TargetBytesPerWord); } } - { object array = makeByteArray + // if (strcmp(name, "avian/VMClass.class") == 0) trap(); + + if (hashMapFind(t, typeMaps, c, objectHash, objectEqual) == 0) { + object array = makeByteArray (t, TypeMap::sizeInBytes (ceiling(classFixedSize(t, c), BytesPerWord), memberIndex)); @@ -384,14 +541,14 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, ceiling(targetMemberOffset, TargetBytesPerWord), memberIndex); for (unsigned i = 0; i < memberIndex; ++i) { - expect(t, buildMemberOffsets[i] + Field* f = memberFields + i; + + expect(t, f->buildOffset < map->buildFixedSizeInWords * BytesPerWord); - map->targetFixedOffsets()[buildMemberOffsets[i]] - = targetMemberOffsets[i]; + map->targetFixedOffsets()[f->buildOffset] = f->targetOffset; - new (map->fixedFields() + i) Field - (memberTypes[i], buildMemberOffsets[i], targetMemberOffsets[i]); + map->fixedFields()[i] = *f; } hashMapInsert(t, typeMaps, c, array, objectHash); @@ -408,20 +565,36 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, TypeMap::SingletonKind); for (unsigned i = 0; i < staticIndex; ++i) { - expect(t, buildStaticOffsets[i] + Field* f = staticFields + i; + + expect(t, f->buildOffset < map->buildFixedSizeInWords * BytesPerWord); - map->targetFixedOffsets()[buildStaticOffsets[i]] - = targetStaticOffsets[i]; + map->targetFixedOffsets()[f->buildOffset] = f->targetOffset; - new (map->fixedFields() + i) Field - (staticTypes[i], buildStaticOffsets[i], targetStaticOffsets[i]); + map->fixedFields()[i] = *f; } hashMapInsert (t, typeMaps, classStaticTable(t, c), array, objectHash); } } + } + } + + for (Finder::Iterator it(finder); it.hasMore();) { + unsigned nameSize = 0; + const char* name = it.next(&nameSize); + + if (endsWith(".class", name, nameSize) + and (className == 0 or strncmp(name, className, nameSize - 6) == 0)) + { + // fprintf(stderr, "%.*s\n", nameSize - 6, name); + object c = resolveSystemClass + (t, root(t, Machine::BootLoader), + makeByteArray(t, "%.*s", nameSize - 6, name), true); + + PROTECT(t, c); if (classMethodTable(t, c)) { for (unsigned i = 0; i < arrayLength(t, classMethodTable(t, c)); ++i) { @@ -444,7 +617,11 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, PROTECT(t, method); t->m->processor->compileMethod - (t, zone, &constants, &calls, &addresses, method); + (t, zone, &constants, &calls, &addresses, method, &resolver); + + if (methodCode(t, method)) { + methods = makePair(t, method, methods); + } } object addendum = methodAddendum(t, method); @@ -499,7 +676,7 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, (reinterpret_cast(code), &location); target_uintptr_t offset = value - code; if (flat) { - offset |= BootFlatConstant; + offset |= TargetBootFlatConstant; } memcpy(location, &offset, TargetBytesPerWord); @@ -510,6 +687,13 @@ makeCodeImage(Thread* t, Zone* zone, BootImage* image, uint8_t* code, - reinterpret_cast(code)); } + for (; methods; methods = pairSecond(t, methods)) { + codeCompiled(t, methodCode(t, pairFirst(t, methods))) + -= reinterpret_cast(code); + } + + t->m->processor->normalizeVirtualThunks(t); + return constants; } @@ -541,17 +725,6 @@ visitRoots(Thread* t, BootImage* image, HeapWalker* w, object constants) } } -TypeMap* -typeMap(Thread* t, object typeMaps, object p) -{ - return reinterpret_cast - (&byteArrayBody - (t, objectClass(t, p) == type(t, Machine::SingletonType) - ? hashMapFind(t, typeMaps, p, objectHash, objectEqual) - : hashMapFind(t, typeMaps, objectClass(t, p), objectHash, objectEqual), - 0)); -} - unsigned targetOffset(Thread* t, object typeMaps, object p, unsigned offset) { @@ -602,6 +775,36 @@ targetSize(Thread* t, object typeMaps, object p) } } +unsigned +objectMaskCount(TypeMap* map) +{ + unsigned count = map->targetFixedSizeInWords; + + if (map->targetArrayElementSizeInBytes) { + ++ count; + } + + return count; +} + +unsigned +targetSize(Thread* t, object typeMaps, object referer, unsigned refererOffset, + object p) +{ + if (referer + and objectClass(t, referer) == type(t, Machine::ClassType) + and (refererOffset * BytesPerWord) == ClassObjectMask) + { + return (TargetBytesPerWord * 2) + + pad + (ceiling + (objectMaskCount + (classTypeMap(t, typeMaps, referer)), 32) * 4, TargetBytesPerWord); + } else { + return targetSize(t, typeMaps, p); + } +} + void copy(Thread* t, uint8_t* src, uint8_t* dst, Type type) { @@ -674,6 +877,7 @@ nonObjectsEqual(uint8_t* src, uint8_t* dst, Type type) return memcmp(dst, src, BytesPerWord) == 0; case Type_object: + case Type_object_nogc: return true; default: abort(); @@ -686,7 +890,7 @@ nonObjectsEqual(TypeMap* map, uint8_t* src, uint8_t* dst) for (unsigned i = 0; i < map->fixedFieldCount; ++i) { Field* field = map->fixedFields() + i; if (not nonObjectsEqual - (src + field->offset, dst + field->targetOffset, field->type)) + (src + field->buildOffset, dst + field->targetOffset, field->type)) { return false; } @@ -720,7 +924,7 @@ copy(Thread* t, object typeMaps, object p, uint8_t* dst) for (unsigned i = 0; i < map->fixedFieldCount; ++i) { Field* field = map->fixedFields() + i; if (field->type > Type_array) abort(t); - copy(t, src + field->offset, dst + field->targetOffset, field->type); + copy(t, src + field->buildOffset, dst + field->targetOffset, field->type); } if (map->targetArrayElementSizeInBytes) { @@ -732,16 +936,59 @@ copy(Thread* t, object typeMaps, object p, uint8_t* dst) dst + (map->targetFixedSizeInWords * TargetBytesPerWord) + (i * map->targetArrayElementSizeInBytes), map->arrayElementType); } + + if (objectClass(t, p) == type(t, Machine::ClassType)) { + uint16_t fixedSize; + uint8_t arrayElementSize; + object array = hashMapFind(t, typeMaps, p, objectHash, objectEqual); + + if (array) { + TypeMap* classMap = reinterpret_cast + (&byteArrayBody(t, array, 0)); + + fixedSize = TARGET_V2 + (classMap->targetFixedSizeInWords * TargetBytesPerWord); + + arrayElementSize = classMap->targetArrayElementSizeInBytes; + } else if (classFixedSize(t, p) == BytesPerWord * 2 + and classArrayElementSize(t, p) == BytesPerWord) + { + fixedSize = TARGET_V2(TargetBytesPerWord * 2); + + arrayElementSize = TargetBytesPerWord; + } else { + fixedSize = 0; + arrayElementSize = 0; + } + + if (fixedSize) { + memcpy(dst + TargetClassFixedSize, &fixedSize, 2); + + memcpy(dst + TargetClassArrayElementSize, &arrayElementSize, 1); + } + + // if (strcmp("vm::lineNumberTable", + // reinterpret_cast(&byteArrayBody(t, className(t, p), 0))) == 0) trap(); + } } else { switch (map->kind) { case TypeMap::NormalKind: + if (objectClass(t, p) == type(t, Machine::FieldType)) { + uint16_t offset = TARGET_V2(targetFieldOffset(t, typeMaps, p)); + memcpy(dst + TargetFieldOffset, &offset, 2); + } break; case TypeMap::SingletonKind: { + unsigned maskSize = singletonMaskSize + (map->targetFixedSizeInWords - 2, TargetBitsPerWord); + + target_uintptr_t targetLength = TARGET_VW + (map->targetFixedSizeInWords - 2 + maskSize); + memcpy(dst + TargetBytesPerWord, &targetLength, TargetBytesPerWord); + uint8_t* mask = dst + (map->targetFixedSizeInWords * TargetBytesPerWord); - memset(mask, 0, singletonMaskSize - (map->targetFixedSizeInWords - 2, TargetBitsPerWord) - * TargetBytesPerWord); + memset(mask, 0, maskSize * TargetBytesPerWord); for (unsigned i = 0; i < map->fixedFieldCount; ++i) { Field* field = map->fixedFields() + i; @@ -766,6 +1013,13 @@ copy(Thread* t, object typeMaps, object p, uint8_t* dst) unsigned poolMaskSize = vm::poolMaskSize (map->targetFixedSizeInWords - 2, TargetBitsPerWord); + unsigned objectMaskSize = singletonMaskSize + (map->targetFixedSizeInWords - 2 + poolMaskSize, TargetBitsPerWord); + + target_uintptr_t targetLength = TARGET_VW + (map->targetFixedSizeInWords - 2 + poolMaskSize + objectMaskSize); + memcpy(dst + TargetBytesPerWord, &targetLength, TargetBytesPerWord); + uint8_t* poolMask = dst + (map->targetFixedSizeInWords * TargetBytesPerWord); @@ -774,9 +1028,7 @@ copy(Thread* t, object typeMaps, object p, uint8_t* dst) uint8_t* objectMask = dst + ((map->targetFixedSizeInWords + poolMaskSize) * TargetBytesPerWord); - memset(objectMask, 0, singletonMaskSize - (map->targetFixedSizeInWords - 2 + poolMaskSize, - TargetBitsPerWord) * TargetBytesPerWord); + memset(objectMask, 0, objectMaskSize * TargetBytesPerWord); for (unsigned i = 0; i < map->fixedFieldCount; ++i) { Field* field = map->fixedFields() + i; @@ -812,21 +1064,64 @@ copy(Thread* t, object typeMaps, object p, uint8_t* dst) default: abort(t); } } +} + +void +copy(Thread* t, object typeMaps, object referer, unsigned refererOffset, + object p, uint8_t* dst) +{ + if (referer + and objectClass(t, referer) == type(t, Machine::ClassType) + and (refererOffset * BytesPerWord) == ClassObjectMask) + { + TypeMap* map = classTypeMap(t, typeMaps, referer); + + memset(dst, 0, TargetBytesPerWord); + + unsigned length = ceiling(objectMaskCount(map), 32); + + target_uintptr_t targetLength = TARGET_VW(length); + + memcpy(dst + TargetBytesPerWord, &targetLength, TargetBytesPerWord); + + memset(dst + (TargetBytesPerWord * 2), 0, length * 4); + + for (unsigned i = 0; i < map->fixedFieldCount; ++i) { + Field* field = map->fixedFields() + i; + if (field->type == Type_object) { + unsigned offset = field->targetOffset / TargetBytesPerWord; + reinterpret_cast(dst + (TargetBytesPerWord * 2)) + [offset / 32] |= static_cast(1) << (offset % 32); + } + } + + if (map->targetArrayElementSizeInBytes + and map->arrayElementType == Type_object) + { + unsigned offset = map->targetFixedSizeInWords; + reinterpret_cast(dst + (TargetBytesPerWord * 2)) + [offset / 32] |= static_cast(1) << (offset % 32); + } + } else { + copy(t, typeMaps, p, dst); + } if (DebugNativeTarget) { expect(t, targetSize(t, typeMaps, p) == baseSize(t, p, objectClass(t, p))); - expect(t, nonObjectsEqual(map, src, dst)); + expect(t, nonObjectsEqual + (typeMap(t, typeMaps, p), reinterpret_cast(p), dst)); } } HeapWalker* -makeHeapImage(Thread* t, BootImage* image, uintptr_t* heap, uintptr_t* map, - unsigned capacity, object constants, object typeMaps) +makeHeapImage(Thread* t, BootImage* image, target_uintptr_t* heap, + target_uintptr_t* map, unsigned capacity, object constants, + object typeMaps) { class Visitor: public HeapVisitor { public: - Visitor(Thread* t, object typeMaps, uintptr_t* heap, - uintptr_t* map, unsigned capacity): + Visitor(Thread* t, object typeMaps, target_uintptr_t* heap, + target_uintptr_t* map, unsigned capacity): t(t), typeMaps(typeMaps), currentObject(0), currentNumber(0), currentOffset(0), heap(heap), map(map), position(0), capacity(capacity) { } @@ -845,8 +1140,8 @@ makeHeapImage(Thread* t, BootImage* image, uintptr_t* heap, uintptr_t* map, (t, typeMaps, currentObject, currentOffset * BytesPerWord) / TargetBytesPerWord); - unsigned mark = heap[offset] & (~PointerMask); - unsigned value = number | (mark << BootShift); + unsigned mark = heap[offset] & (~TargetPointerMask); + unsigned value = number | (mark << TargetBootShift); if (value) markBit(map, offset); @@ -860,7 +1155,8 @@ makeHeapImage(Thread* t, BootImage* image, uintptr_t* heap, uintptr_t* map, virtual unsigned visitNew(object p) { if (p) { - unsigned size = targetSize(t, typeMaps, p); + unsigned size = targetSize + (t, typeMaps, currentObject, currentOffset, p); unsigned number; if ((currentObject @@ -877,7 +1173,7 @@ makeHeapImage(Thread* t, BootImage* image, uintptr_t* heap, uintptr_t* map, target_uintptr_t* dst = heap + position + TargetFixieSizeInWords; - unsigned maskSize = ceiling(size, TargetBytesPerWord); + unsigned maskSize = ceiling(size, TargetBitsPerWord); unsigned total = TargetFixieSizeInWords + size + maskSize; @@ -897,7 +1193,8 @@ makeHeapImage(Thread* t, BootImage* image, uintptr_t* heap, uintptr_t* map, memcpy(reinterpret_cast(heap + position) + TargetFixieSize, &targetSize, 4); - copy(t, typeMaps, p, reinterpret_cast(dst)); + copy(t, typeMaps, currentObject, currentOffset, p, + reinterpret_cast(dst)); dst[0] |= FixedMark; @@ -909,7 +1206,8 @@ makeHeapImage(Thread* t, BootImage* image, uintptr_t* heap, uintptr_t* map, } else { expect(t, position + size < capacity); - copy(t, typeMaps, p, reinterpret_cast(heap + position)); + copy(t, typeMaps, currentObject, currentOffset, p, + reinterpret_cast(heap + position)); number = position + 1; position += size; @@ -970,15 +1268,21 @@ updateConstants(Thread* t, object constants, uint8_t* code, uintptr_t* codeMap, { void* location; bool flat = pl->resolve(0, &location); - target_uintptr_t offset = target | BootHeapOffset; + target_uintptr_t offset = target | TargetBootHeapOffset; if (flat) { - offset |= BootFlatConstant; + offset |= TargetBootFlatConstant; } memcpy(location, &offset, TargetBytesPerWord); expect(t, reinterpret_cast(location) >= reinterpret_cast(code)); + // fprintf(stderr, "mark constant %d %d\n", + // static_cast + // (reinterpret_cast(location) + // - reinterpret_cast(code)), + // static_cast(offset)); + markBit(codeMap, reinterpret_cast(location) - reinterpret_cast(code)); } @@ -1014,12 +1318,6 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, typeMaps = makeHashMap(t, 0, 0); PROTECT(t, typeMaps); - constants = makeCodeImage - (t, &zone, image, code, codeMap, className, methodName, methodSpec, - typeMaps); - - PROTECT(t, constants); - #include "type-maps.cpp" for (unsigned i = 0; i < arrayLength(t, t->m->types); ++i) { @@ -1030,21 +1328,26 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, } ++ count; - Type types[count]; - types[0] = Type_object; - unsigned buildOffsets[count]; - buildOffsets[0] = 0; + Field fields[count]; + + new (fields) Field(Type_object, 0, BytesPerWord, 0, TargetBytesPerWord); + unsigned buildOffset = BytesPerWord; - unsigned targetOffsets[count]; - targetOffsets[0] = 0; unsigned targetOffset = TargetBytesPerWord; bool sawArray = false; - unsigned buildSize = BytesPerWord; - unsigned targetSize = TargetBytesPerWord; + Type type; + unsigned buildSize; + unsigned targetSize; for (unsigned j = 1; j < count; ++j) { switch (source[j - 1]) { case Type_object: - types[j] = Type_object; + type = Type_object; + buildSize = BytesPerWord; + targetSize = TargetBytesPerWord; + break; + + case Type_object_nogc: + type = Type_object_nogc; buildSize = BytesPerWord; targetSize = TargetBytesPerWord; break; @@ -1052,39 +1355,39 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, case Type_word: case Type_intptr_t: case Type_uintptr_t: - types[j] = Type_intptr_t; + type = Type_intptr_t; buildSize = BytesPerWord; targetSize = TargetBytesPerWord; break; case Type_int8_t: case Type_uint8_t: - types[j] = Type_int8_t; + type = Type_int8_t; buildSize = targetSize = 1; break; case Type_int16_t: case Type_uint16_t: - types[j] = Type_int16_t; + type = Type_int16_t; buildSize = targetSize = 2; break; case Type_int32_t: case Type_uint32_t: case Type_float: - types[j] = Type_int32_t; + type = Type_int32_t; buildSize = targetSize = 4; break; case Type_int64_t: case Type_uint64_t: case Type_double: - types[j] = Type_int64_t; + type = Type_int64_t; buildSize = targetSize = 8; break; case Type_array: - types[j] = Type_none; + type = Type_none; buildSize = targetSize = 0; break; @@ -1100,16 +1403,14 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, ++ buildOffset; } - buildOffsets[j] = buildOffset; - - buildOffset += buildSize; - while (targetOffset % targetSize) { ++ targetOffset; } - targetOffsets[j] = targetOffset; + new (fields + j) Field + (type, buildOffset, buildSize, targetOffset, targetSize); + buildOffset += buildSize; targetOffset += targetSize; } } @@ -1120,7 +1421,7 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, unsigned targetArrayElementSize; if (sawArray) { fixedFieldCount = count - 2; - arrayElementType = types[count - 1]; + arrayElementType = type; buildArrayElementSize = buildSize; targetArrayElementSize = targetSize; } else { @@ -1141,19 +1442,27 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, targetArrayElementSize, arrayElementType); for (unsigned j = 0; j < fixedFieldCount; ++j) { - expect(t, buildOffsets[j] < map->buildFixedSizeInWords * BytesPerWord); + Field* f = fields + j; - map->targetFixedOffsets()[buildOffsets[j]] = targetOffsets[j]; + expect(t, f->buildOffset + < map->buildFixedSizeInWords * BytesPerWord); - new (map->fixedFields() + j) Field - (types[j], buildOffsets[j], targetOffsets[j]); + map->targetFixedOffsets()[f->buildOffset] = f->targetOffset; + + map->fixedFields()[j] = *f; } - hashMapInsertOrReplace - (t, typeMaps, type(t, static_cast(i)), array, - objectHash, objectEqual); + hashMapInsert + (t, typeMaps, vm::type(t, static_cast(i)), array, + objectHash); } + constants = makeCodeImage + (t, &zone, image, code, codeMap, className, methodName, methodSpec, + typeMaps); + + PROTECT(t, constants); + // these roots will not be used when the bootimage is loaded, so // there's no need to preserve them: setRoot(t, Machine::PoolMap, 0); @@ -1217,9 +1526,10 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, } } - uintptr_t* heap = static_cast + target_uintptr_t* heap = static_cast (t->m->heap->allocate(HeapCapacity)); - uintptr_t* heapMap = static_cast + + target_uintptr_t* heapMap = static_cast (t->m->heap->allocate(heapMapSize(HeapCapacity))); memset(heapMap, 0, heapMapSize(HeapCapacity)); @@ -1275,7 +1585,6 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, heapWalker->dispose(); image->magic = BootImage::Magic; - image->codeBase = reinterpret_cast(code); fprintf(stderr, "class count %d string count %d call count %d\n" "heap size %d code size %d\n", @@ -1301,11 +1610,15 @@ writeBootImage2(Thread* t, FILE* out, BootImage* image, uint8_t* code, ++ offset; } - fwrite(heapMap, pad(heapMapSize(image->heapSize)), 1, out); - fwrite(heap, pad(image->heapSize), 1, out); + fwrite + (heapMap, pad(heapMapSize(image->heapSize), TargetBytesPerWord), 1, out); - fwrite(codeMap, pad(codeMapSize(image->codeSize)), 1, out); - fwrite(code, pad(image->codeSize), 1, out); + fwrite(heap, pad(image->heapSize, TargetBytesPerWord), 1, out); + + fwrite + (codeMap, pad(codeMapSize(image->codeSize), TargetBytesPerWord), 1, out); + + fwrite(code, pad(image->codeSize, TargetBytesPerWord), 1, out); } } diff --git a/src/bootimage.h b/src/bootimage.h index a5ed58bb61..fa3badac08 100644 --- a/src/bootimage.h +++ b/src/bootimage.h @@ -12,16 +12,11 @@ #define BOOTIMAGE_H #include "common.h" +#include "target.h" +#include "machine.h" namespace vm { -const unsigned BootMask = (~static_cast(0)) / BytesPerWord; - -const unsigned BootShift = 32 - log(BytesPerWord); - -const unsigned BootFlatConstant = 1 << BootShift; -const unsigned BootHeapOffset = 1 << (BootShift + 1); - class BootImage { public: class Thunk { @@ -30,14 +25,14 @@ class BootImage { start(0), frameSavedOffset(0), length(0) { } - Thunk(unsigned start, unsigned frameSavedOffset, unsigned length): + Thunk(uint32_t start, uint32_t frameSavedOffset, uint32_t length): start(start), frameSavedOffset(frameSavedOffset), length(length) { } - unsigned start; - unsigned frameSavedOffset; - unsigned length; - }; + uint32_t start; + uint32_t frameSavedOffset; + uint32_t length; + } PACKED; class ThunkCollection { public: @@ -47,63 +42,56 @@ class BootImage { Thunk aioob; Thunk stackOverflow; Thunk table; - }; + } PACKED; - static const unsigned Magic = 0x22377322; + static const uint32_t Magic = 0x22377322; - unsigned magic; + uint32_t magic; - unsigned heapSize; - unsigned codeSize; + uint32_t heapSize; + uint32_t codeSize; - unsigned bootClassCount; - unsigned appClassCount; - unsigned stringCount; - unsigned callCount; + uint32_t bootClassCount; + uint32_t appClassCount; + uint32_t stringCount; + uint32_t callCount; - unsigned bootLoader; - unsigned appLoader; - unsigned types; - unsigned methodTree; - unsigned methodTreeSentinal; - unsigned virtualThunks; + uint32_t bootLoader; + uint32_t appLoader; + uint32_t types; + uint32_t methodTree; + uint32_t methodTreeSentinal; + uint32_t virtualThunks; - uintptr_t codeBase; + uint32_t compileMethodCall; + uint32_t compileVirtualMethodCall; + uint32_t invokeNativeCall; + uint32_t throwArrayIndexOutOfBoundsCall; + uint32_t throwStackOverflowCall; - ThunkCollection thunks; - - unsigned compileMethodCall; - unsigned compileVirtualMethodCall; - unsigned invokeNativeCall; - unsigned throwArrayIndexOutOfBoundsCall; - unsigned throwStackOverflowCall; - -#define THUNK(s) unsigned s##Call; +#define THUNK(s) uint32_t s##Call; #include "thunks.cpp" #undef THUNK + + ThunkCollection thunks; +} PACKED; + +class OffsetResolver { + public: + virtual unsigned fieldOffset(Thread*, object) = 0; }; -inline unsigned -codeMapSize(unsigned codeSize) -{ - return ceiling(codeSize, BitsPerWord) * BytesPerWord; -} +#define NAME(x) Target##x +#define LABEL(x) target_##x +#include "bootimage-template.cpp" +#undef LABEL +#undef NAME -inline unsigned -heapMapSize(unsigned heapSize) -{ - return ceiling(heapSize, BitsPerWord * BytesPerWord) * BytesPerWord; -} - -inline object -bootObject(uintptr_t* heap, unsigned offset) -{ - if (offset) { - return reinterpret_cast(heap + offset - 1); - } else { - return 0; - } -} +#define NAME(x) x +#define LABEL(x) x +#include "bootimage-template.cpp" +#undef LABEL +#undef NAME } // namespace vm diff --git a/src/common.h b/src/common.h index 6a88c6cfa8..118007db47 100644 --- a/src/common.h +++ b/src/common.h @@ -45,6 +45,8 @@ typedef unsigned __int64 uint64_t; # define NO_RETURN __declspec(noreturn) +# define PACKED + # define PLATFORM_WINDOWS # ifdef _M_IX86 @@ -76,6 +78,8 @@ typedef intptr_t intptr_alias_t; # define NO_RETURN __attribute__((noreturn)) +# define PACKED __attribute__((packed)) + # ifdef __MINGW32__ # define PLATFORM_WINDOWS # endif diff --git a/src/compile.cpp b/src/compile.cpp index fa1ffa4778..0dbcb854cc 100644 --- a/src/compile.cpp +++ b/src/compile.cpp @@ -336,7 +336,7 @@ setRoot(Thread* t, Root root, object value); unsigned compiledSize(intptr_t address) { - return reinterpret_cast(address)[-1]; + return reinterpret_cast(address)[-1]; } intptr_t @@ -956,9 +956,10 @@ class BootContext { }; BootContext(Thread* t, object constants, object calls, - DelayedPromise* addresses, Zone* zone): + DelayedPromise* addresses, Zone* zone, OffsetResolver* resolver): protector(t, this), constants(constants), calls(calls), - addresses(addresses), addressSentinal(addresses), zone(zone) + addresses(addresses), addressSentinal(addresses), zone(zone), + resolver(resolver) { } MyProtector protector; @@ -967,6 +968,7 @@ class BootContext { DelayedPromise* addresses; DelayedPromise* addressSentinal; Zone* zone; + OffsetResolver* resolver; }; class Context { @@ -3309,7 +3311,7 @@ compileDirectInvoke(MyThread* t, Frame* frame, object target, bool tailCall, (TargetBytesPerWord, frame->addressOperand(returnAddressPromise), TargetBytesPerWord, c->memory (c->register_(t->arch->thread()), Compiler::AddressType, - difference(&(t->tailAddress), t))); + TargetThreadTailAddress)); c->exit (c->constant @@ -3800,6 +3802,16 @@ intrinsic(MyThread* t, Frame* frame, object target) return false; } +unsigned +targetFieldOffset(Context* context, object field) +{ + if (context->bootContext) { + return context->bootContext->resolver->fieldOffset(context->thread, field); + } else { + return fieldOffset(context->thread, field); + } +} + void compile(MyThread* t, Frame* initialFrame, unsigned ip, int exceptionHandlerStart) @@ -3861,7 +3873,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } if (CheckArrayBounds) { - c->checkBounds(array, ArrayLength, index, aioobThunk(t)); + c->checkBounds(array, TargetArrayLength, index, aioobThunk(t)); } switch (instruction) { @@ -3869,7 +3881,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, frame->pushObject (c->load (TargetBytesPerWord, TargetBytesPerWord, c->memory - (array, Compiler::ObjectType, ArrayBody, index, + (array, Compiler::ObjectType, TargetArrayBody, index, TargetBytesPerWord), TargetBytesPerWord)); break; @@ -3878,7 +3890,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, frame->pushInt (c->load (4, 4, c->memory - (array, Compiler::FloatType, ArrayBody, index, 4), + (array, Compiler::FloatType, TargetArrayBody, index, 4), TargetBytesPerWord)); break; @@ -3886,7 +3898,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, frame->pushInt (c->load (4, 4, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 4), + (array, Compiler::IntegerType, TargetArrayBody, index, 4), TargetBytesPerWord)); break; @@ -3894,7 +3906,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, frame->pushInt (c->load (1, 1, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 1), + (array, Compiler::IntegerType, TargetArrayBody, index, 1), TargetBytesPerWord)); break; @@ -3902,7 +3914,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, frame->pushInt (c->loadz (2, 2, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 2), + (array, Compiler::IntegerType, TargetArrayBody, index, 2), TargetBytesPerWord)); break; @@ -3910,21 +3922,21 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, frame->pushLong (c->load (8, 8, c->memory - (array, Compiler::FloatType, ArrayBody, index, 8), 8)); + (array, Compiler::FloatType, TargetArrayBody, index, 8), 8)); break; case laload: frame->pushLong (c->load (8, 8, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 8), 8)); + (array, Compiler::IntegerType, TargetArrayBody, index, 8), 8)); break; case saload: frame->pushInt (c->load (2, 2, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 2), + (array, Compiler::IntegerType, TargetArrayBody, index, 2), TargetBytesPerWord)); break; } @@ -3956,7 +3968,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, } if (CheckArrayBounds) { - c->checkBounds(array, ArrayLength, index, aioobThunk(t)); + c->checkBounds(array, TargetArrayLength, index, aioobThunk(t)); } switch (instruction) { @@ -3969,7 +3981,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, Compiler::VoidType, 4, c->register_(t->arch->thread()), array, c->add - (4, c->constant(ArrayBody, Compiler::IntegerType), + (4, c->constant(TargetArrayBody, Compiler::IntegerType), c->shl (4, c->constant(log(TargetBytesPerWord), Compiler::IntegerType), index)), @@ -3979,38 +3991,38 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, case fastore: c->store (TargetBytesPerWord, value, 4, c->memory - (array, Compiler::FloatType, ArrayBody, index, 4)); + (array, Compiler::FloatType, TargetArrayBody, index, 4)); break; case iastore: c->store (TargetBytesPerWord, value, 4, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 4)); + (array, Compiler::IntegerType, TargetArrayBody, index, 4)); break; case bastore: c->store (TargetBytesPerWord, value, 1, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 1)); + (array, Compiler::IntegerType, TargetArrayBody, index, 1)); break; case castore: case sastore: c->store (TargetBytesPerWord, value, 2, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 2)); + (array, Compiler::IntegerType, TargetArrayBody, index, 2)); break; case dastore: c->store (8, value, 8, c->memory - (array, Compiler::FloatType, ArrayBody, index, 8)); + (array, Compiler::FloatType, TargetArrayBody, index, 8)); break; case lastore: c->store (8, value, 8, c->memory - (array, Compiler::IntegerType, ArrayBody, index, 8)); + (array, Compiler::IntegerType, TargetArrayBody, index, 8)); break; } } break; @@ -4082,7 +4094,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, (c->load (TargetBytesPerWord, TargetBytesPerWord, c->memory - (frame->popObject(), Compiler::IntegerType, ArrayLength, 0, 1), + (frame->popObject(), Compiler::IntegerType, + TargetArrayLength, 0, 1), TargetBytesPerWord)); } break; @@ -4425,54 +4438,56 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, frame->pushInt (c->load (1, 1, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1), - TargetBytesPerWord)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1), TargetBytesPerWord)); break; case CharField: frame->pushInt (c->loadz (2, 2, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1), - TargetBytesPerWord)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1), TargetBytesPerWord)); break; case ShortField: frame->pushInt (c->load (2, 2, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1), - TargetBytesPerWord)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1), TargetBytesPerWord)); break; case FloatField: frame->pushInt (c->load (4, 4, c->memory - (table, Compiler::FloatType, fieldOffset(t, field), 0, 1), - TargetBytesPerWord)); + (table, Compiler::FloatType, targetFieldOffset + (context, field), 0, 1), TargetBytesPerWord)); break; case IntField: frame->pushInt (c->load (4, 4, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1), - TargetBytesPerWord)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1), TargetBytesPerWord)); break; case DoubleField: frame->pushLong (c->load (8, 8, c->memory - (table, Compiler::FloatType, fieldOffset(t, field), 0, 1), 8)); + (table, Compiler::FloatType, targetFieldOffset + (context, field), 0, 1), 8)); break; case LongField: frame->pushLong (c->load (8, 8, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1), 8)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1), 8)); break; case ObjectField: @@ -4480,8 +4495,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, (c->load (TargetBytesPerWord, TargetBytesPerWord, c->memory - (table, Compiler::ObjectType, fieldOffset(t, field), 0, 1), - TargetBytesPerWord)); + (table, Compiler::ObjectType, targetFieldOffset + (context, field), 0, 1), TargetBytesPerWord)); break; default: @@ -4974,8 +4989,8 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, if (LIKELY(methodVirtual(t, target))) { unsigned parameterFootprint = methodParameterFootprint(t, target); - unsigned offset = ClassVtable - + (methodOffset(t, target) * BytesPerWord); + unsigned offset = TargetClassVtable + + (methodOffset(t, target) * TargetBytesPerWord); Compiler::Operand* instance = c->peek(1, parameterFootprint - 1); @@ -4985,7 +5000,7 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, (c->memory (c->and_ (TargetBytesPerWord, c->constant - (PointerMask, Compiler::IntegerType), + (TargetPointerMask, Compiler::IntegerType), c->memory(instance, Compiler::ObjectType, 0, 0, 1)), Compiler::ObjectType, offset, 0, 1), tailCall ? Compiler::TailJump : 0, @@ -5635,38 +5650,44 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, case BooleanField: c->store (TargetBytesPerWord, value, 1, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1)); break; case CharField: case ShortField: c->store (TargetBytesPerWord, value, 2, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1)); break; case FloatField: c->store (TargetBytesPerWord, value, 4, c->memory - (table, Compiler::FloatType, fieldOffset(t, field), 0, 1)); + (table, Compiler::FloatType, targetFieldOffset + (context, field), 0, 1)); break; case IntField: c->store (TargetBytesPerWord, value, 4, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1)); break; case DoubleField: c->store (8, value, 8, c->memory - (table, Compiler::FloatType, fieldOffset(t, field), 0, 1)); + (table, Compiler::FloatType, targetFieldOffset + (context, field), 0, 1)); break; case LongField: c->store (8, value, 8, c->memory - (table, Compiler::IntegerType, fieldOffset(t, field), 0, 1)); + (table, Compiler::IntegerType, targetFieldOffset + (context, field), 0, 1)); break; case ObjectField: @@ -5679,14 +5700,16 @@ compile(MyThread* t, Frame* initialFrame, unsigned ip, 0, Compiler::VoidType, 4, c->register_(t->arch->thread()), table, - c->constant(fieldOffset(t, field), Compiler::IntegerType), + c->constant(targetFieldOffset(context, field), + Compiler::IntegerType), value); } else { c->call (c->constant(getThunk(t, setThunk), Compiler::AddressType), 0, 0, 0, Compiler::VoidType, 4, c->register_(t->arch->thread()), table, - c->constant(fieldOffset(t, field), Compiler::IntegerType), + c->constant(targetFieldOffset(context, field), + Compiler::IntegerType), value); } break; @@ -6797,7 +6820,7 @@ finish(MyThread* t, FixedAllocator* allocator, Context* context) // of cycles if another thread compiles the same method in parallel, // which might be mitigated by fine-grained, per-method locking): c->compile(context->leaf ? 0 : stackOverflowThunk(t), - difference(&(t->stackLimit), t)); + TargetThreadStackLimit); // we must acquire the class lock here at the latest @@ -6806,7 +6829,8 @@ finish(MyThread* t, FixedAllocator* allocator, Context* context) unsigned total = pad(codeSize) + pad(c->poolSize()) + TargetBytesPerWord; - uintptr_t* code = static_cast(allocator->allocate(total)); + target_uintptr_t* code = static_cast + (allocator->allocate(total)); code[0] = codeSize; uint8_t* start = reinterpret_cast(code + 1); @@ -6817,8 +6841,7 @@ finish(MyThread* t, FixedAllocator* allocator, Context* context) if (context->objectPool) { object pool = allocate3 (t, allocator, Machine::ImmortalAllocation, - FixedSizeOfArray - + ((context->objectPoolCount + 1) * TargetBytesPerWord), + FixedSizeOfArray + ((context->objectPoolCount + 1) * BytesPerWord), true); initArray(t, pool, context->objectPoolCount + 1); @@ -6829,7 +6852,7 @@ finish(MyThread* t, FixedAllocator* allocator, Context* context) unsigned i = 1; for (PoolElement* p = context->objectPool; p; p = p->next) { - unsigned offset = ArrayBody + ((i++) * TargetBytesPerWord); + unsigned offset = ArrayBody + ((i++) * BytesPerWord); p->address = reinterpret_cast(pool) + offset; @@ -8318,13 +8341,30 @@ class MyProcessor: public Processor { t->init(); if (false) { - fprintf(stderr, "%d\n", difference(&(t->stack), t)); - fprintf(stderr, "%d\n", difference(&(t->scratch), t)); - fprintf(stderr, "%d\n", difference(&(t->continuation), t)); - fprintf(stderr, "%d\n", difference(&(t->exception), t)); - fprintf(stderr, "%d\n", difference(&(t->exceptionStackAdjustment), t)); - fprintf(stderr, "%d\n", difference(&(t->exceptionOffset), t)); - fprintf(stderr, "%d\n", difference(&(t->exceptionHandler), t)); + fprintf(stderr, "stack %d\n", + difference(&(t->stack), t)); + fprintf(stderr, "scratch %d\n", + difference(&(t->scratch), t)); + fprintf(stderr, "continuation %d\n", + difference(&(t->continuation), t)); + fprintf(stderr, "exception %d\n", + difference(&(t->exception), t)); + fprintf(stderr, "exceptionStackAdjustment %d\n", + difference(&(t->exceptionStackAdjustment), t)); + fprintf(stderr, "exceptionOffset %d\n", + difference(&(t->exceptionOffset), t)); + fprintf(stderr, "exceptionHandler %d\n", + difference(&(t->exceptionHandler), t)); + fprintf(stderr, "tailAddress %d\n", + difference(&(t->tailAddress), t)); + fprintf(stderr, "stackLimit %d\n", + difference(&(t->stackLimit), t)); + fprintf(stderr, "ip %d\n", + difference(&(t->ip), t)); + fprintf(stderr, "virtualCallTarget %d\n", + difference(&(t->virtualCallTarget), t)); + fprintf(stderr, "virtualCallIndex %d\n", + difference(&(t->virtualCallIndex), t)); exit(0); } @@ -8671,10 +8711,10 @@ class MyProcessor: public Processor { virtual void compileMethod(Thread* vmt, Zone* zone, object* constants, object* calls, DelayedPromise** addresses, - object method) + object method, OffsetResolver* resolver) { MyThread* t = static_cast(vmt); - BootContext bootContext(t, *constants, *calls, *addresses, zone); + BootContext bootContext(t, *constants, *calls, *addresses, zone, resolver); compile(t, &codeAllocator, &bootContext, method); @@ -8689,6 +8729,17 @@ class MyProcessor: public Processor { bootImage->virtualThunks = w->visitRoot(root(t, VirtualThunks)); } + virtual void normalizeVirtualThunks(Thread* t) { + for (unsigned i = 0; i < wordArrayLength(t, root(t, VirtualThunks)); + i += 2) + { + if (wordArrayBody(t, root(t, VirtualThunks), i)) { + wordArrayBody(t, root(t, VirtualThunks), i) + -= reinterpret_cast(codeAllocator.base); + } + } + } + virtual unsigned* makeCallTable(Thread* t, HeapWalker* w) { bootImage->codeSize = codeAllocator.offset; bootImage->callCount = callTableSize; @@ -8704,7 +8755,7 @@ class MyProcessor: public Processor { table[index++] = callNodeAddress(t, p) - reinterpret_cast(codeAllocator.base); table[index++] = w->map()->find(callNodeTarget(t, p)) - | (static_cast(callNodeFlags(t, p)) << BootShift); + | (static_cast(callNodeFlags(t, p)) << TargetBootShift); } } @@ -8989,7 +9040,8 @@ insertCallNode(MyThread* t, object node) } object -makeClassMap(Thread* t, unsigned* table, unsigned count, uintptr_t* heap) +makeClassMap(Thread* t, unsigned* table, unsigned count, + uintptr_t* heap) { object array = makeArray(t, nextPowerOfTwo(count)); object map = makeHashMap(t, 0, array); @@ -9068,6 +9120,7 @@ fixupHeap(MyThread* t UNUSED, uintptr_t* map, unsigned size, uintptr_t* heap) for (unsigned bit = 0; bit < BitsPerWord; ++bit) { if (w & (static_cast(1) << bit)) { unsigned index = indexOf(word, bit); + uintptr_t* p = heap + index; assert(t, *p); @@ -9103,6 +9156,10 @@ fixupCode(Thread* t, uintptr_t* map, unsigned size, uint8_t* code, if (oldValue & BootHeapOffset) { newValue = reinterpret_cast (heap + (oldValue & BootMask) - 1); + + // fprintf(stderr, "constant marked %d %d\n", + // index, static_cast(oldValue)); + } else { newValue = reinterpret_cast (code + (oldValue & BootMask)); @@ -9130,12 +9187,11 @@ fixupMethods(Thread* t, object map, BootImage* image, uint8_t* code) for (unsigned i = 0; i < arrayLength(t, classMethodTable(t, c)); ++i) { object method = arrayBody(t, classMethodTable(t, c), i); if (methodCode(t, method)) { - assert(t, (methodCompiled(t, method) - image->codeBase) - <= image->codeSize); + assert(t, methodCompiled(t, method) + <= static_cast(image->codeSize)); codeCompiled(t, methodCode(t, method)) - = (methodCompiled(t, method) - image->codeBase) - + reinterpret_cast(code); + = methodCompiled(t, method) + reinterpret_cast(code); if (DebugCompile) { logCompile @@ -9203,13 +9259,13 @@ fixupThunks(MyThread* t, BootImage* image, uint8_t* code) } void -fixupVirtualThunks(MyThread* t, BootImage* image, uint8_t* code) +fixupVirtualThunks(MyThread* t, uint8_t* code) { for (unsigned i = 0; i < wordArrayLength(t, root(t, VirtualThunks)); i += 2) { if (wordArrayBody(t, root(t, VirtualThunks), i)) { wordArrayBody(t, root(t, VirtualThunks), i) - = (wordArrayBody(t, root(t, VirtualThunks), i) - image->codeBase) + = wordArrayBody(t, root(t, VirtualThunks), i) + reinterpret_cast(code); } } @@ -9231,16 +9287,16 @@ boot(MyThread* t, BootImage* image) (heapMapSize(image->heapSize), BytesPerWord); uintptr_t* heap = heapMap + heapMapSizeInWords; -// fprintf(stderr, "heap from %p to %p\n", -// heap, heap + ceiling(image->heapSize, BytesPerWord)); + // fprintf(stderr, "heap from %p to %p\n", + // heap, heap + ceiling(image->heapSize, BytesPerWord)); uintptr_t* codeMap = heap + ceiling(image->heapSize, BytesPerWord); unsigned codeMapSizeInWords = ceiling (codeMapSize(image->codeSize), BytesPerWord); uint8_t* code = reinterpret_cast(codeMap + codeMapSizeInWords); -// fprintf(stderr, "code from %p to %p\n", -// code, code + image->codeSize); + // fprintf(stderr, "code from %p to %p\n", + // code, code + image->codeSize); fixupHeap(t, heapMap, heapMapSizeInWords, heap); @@ -9293,7 +9349,7 @@ boot(MyThread* t, BootImage* image) fixupThunks(t, image, code); - fixupVirtualThunks(t, image, code); + fixupVirtualThunks(t, code); fixupMethods (t, classLoaderMap(t, root(t, Machine::BootLoader)), image, code); @@ -9337,7 +9393,7 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p) { Assembler* a = defaultContext.context.assembler; - a->saveFrame(difference(&(t->stack), t), difference(&(t->ip), t)); + a->saveFrame(TargetThreadStack, TargetThreadIp); p->thunks.default_.frameSavedOffset = a->length(); @@ -9369,19 +9425,19 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p) TargetBytesPerWord, RegisterOperand, &class_); Assembler::Memory virtualCallTargetDst - (t->arch->thread(), difference(&(t->virtualCallTarget), t)); + (t->arch->thread(), TargetThreadVirtualCallTarget); a->apply(Move, TargetBytesPerWord, RegisterOperand, &class_, TargetBytesPerWord, MemoryOperand, &virtualCallTargetDst); Assembler::Register index(t->arch->virtualCallIndex()); Assembler::Memory virtualCallIndex - (t->arch->thread(), difference(&(t->virtualCallIndex), t)); + (t->arch->thread(), TargetThreadVirtualCallIndex); a->apply(Move, TargetBytesPerWord, RegisterOperand, &index, TargetBytesPerWord, MemoryOperand, &virtualCallIndex); - a->saveFrame(difference(&(t->stack), t), difference(&(t->ip), t)); + a->saveFrame(TargetThreadStack, TargetThreadIp); p->thunks.defaultVirtual.frameSavedOffset = a->length(); @@ -9403,7 +9459,7 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p) { Assembler* a = nativeContext.context.assembler; - a->saveFrame(difference(&(t->stack), t), difference(&(t->ip), t)); + a->saveFrame(TargetThreadStack, TargetThreadIp); p->thunks.native.frameSavedOffset = a->length(); @@ -9414,7 +9470,7 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p) a->apply(LongCall, TargetBytesPerWord, ConstantOperand, &proc); a->popFrameAndUpdateStackAndReturn - (t->arch->alignFrameSize(1), difference(&(t->stack), t)); + (t->arch->alignFrameSize(1), TargetThreadStack); p->thunks.native.length = a->endBlock(false)->resolve(0, 0); } @@ -9423,7 +9479,7 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p) { Assembler* a = aioobContext.context.assembler; - a->saveFrame(difference(&(t->stack), t), difference(&(t->ip), t)); + a->saveFrame(TargetThreadStack, TargetThreadIp); p->thunks.aioob.frameSavedOffset = a->length(); @@ -9440,7 +9496,7 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p) { Assembler* a = stackOverflowContext.context.assembler; - a->saveFrame(difference(&(t->stack), t), difference(&(t->ip), t)); + a->saveFrame(TargetThreadStack, TargetThreadIp); p->thunks.stackOverflow.frameSavedOffset = a->length(); @@ -9457,7 +9513,7 @@ compileThunks(MyThread* t, Allocator* allocator, MyProcessor* p) { Assembler* a = tableContext.context.assembler; - a->saveFrame(difference(&(t->stack), t), difference(&(t->ip), t)); + a->saveFrame(TargetThreadStack, TargetThreadIp); p->thunks.table.frameSavedOffset = a->length(); diff --git a/src/compiler.cpp b/src/compiler.cpp index 157bb82d61..5221485cac 100644 --- a/src/compiler.cpp +++ b/src/compiler.cpp @@ -469,7 +469,7 @@ class PoolPromise: public Promise { virtual int64_t value() { if (resolved()) { return reinterpret_cast - (c->machineCode + pad(c->machineCodeSize) + (c->machineCode + pad(c->machineCodeSize, TargetBytesPerWord) + (key * TargetBytesPerWord)); } @@ -6949,15 +6949,15 @@ class MyCompiler: public Compiler { int i = 0; for (ConstantPoolNode* n = c.firstConstant; n; n = n->next) { - intptr_t* target = reinterpret_cast - (c.machineCode + pad(c.machineCodeSize) + i); + target_intptr_t* target = reinterpret_cast + (c.machineCode + pad(c.machineCodeSize, TargetBytesPerWord) + i); if (n->promise->resolved()) { *target = n->promise->value(); } else { class Listener: public Promise::Listener { public: - Listener(intptr_t* target): target(target){ } + Listener(target_intptr_t* target): target(target){ } virtual bool resolve(int64_t value, void** location) { *target = value; @@ -6965,7 +6965,7 @@ class MyCompiler: public Compiler { return true; } - intptr_t* target; + target_intptr_t* target; }; new (n->promise->listen(sizeof(Listener))) Listener(target); } diff --git a/src/heap.cpp b/src/heap.cpp index 8208cd7e73..c8f30c0369 100644 --- a/src/heap.cpp +++ b/src/heap.cpp @@ -529,7 +529,7 @@ class Fixie { } static unsigned maskSize(unsigned size, bool hasMask) { - return hasMask * ceiling(size, BytesPerWord) * BytesPerWord; + return hasMask * ceiling(size, BitsPerWord) * BytesPerWord; } static unsigned totalSize(unsigned size, bool hasMask) { diff --git a/src/interpret.cpp b/src/interpret.cpp index 255977c25d..45a3105753 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -3067,7 +3067,7 @@ class MyProcessor: public Processor { } virtual void compileMethod(vm::Thread*, Zone*, object*, object*, - DelayedPromise**, object) + DelayedPromise**, object, OffsetResolver*) { abort(s); } @@ -3076,6 +3076,10 @@ class MyProcessor: public Processor { abort(s); } + virtual void normalizeVirtualThunks(vm::Thread*) { + abort(s); + } + virtual unsigned* makeCallTable(vm::Thread*, HeapWalker*) { abort(s); } diff --git a/src/processor.h b/src/processor.h index f790491439..4163590846 100644 --- a/src/processor.h +++ b/src/processor.h @@ -122,11 +122,15 @@ class Processor { virtual void compileMethod(Thread* t, Zone* zone, object* constants, object* calls, - DelayedPromise** addresses, object method) = 0; + DelayedPromise** addresses, object method, + OffsetResolver* resolver) = 0; virtual void visitRoots(Thread* t, HeapWalker* w) = 0; + virtual void + normalizeVirtualThunks(Thread* t) = 0; + virtual unsigned* makeCallTable(Thread* t, HeapWalker* w) = 0; diff --git a/src/target.h b/src/target.h index 91244b7ed6..ec309c8cd0 100644 --- a/src/target.h +++ b/src/target.h @@ -42,25 +42,62 @@ namespace vm { #ifdef TARGET_BYTES_PER_WORD # if (TARGET_BYTES_PER_WORD == 8) # define TARGET_VW(v) TARGET_V8(v) + typedef uint64_t target_uintptr_t; typedef int64_t target_intptr_t; + const unsigned TargetBytesPerWord = 8; + +const unsigned TargetThreadTailAddress = 2272; +const unsigned TargetThreadStackLimit = 2336; +const unsigned TargetThreadStack = 2224; +const unsigned TargetThreadIp = 2216; +const unsigned TargetThreadVirtualCallTarget = 2280; +const unsigned TargetThreadVirtualCallIndex = 2288; + +const unsigned TargetClassFixedSize = 12; +const unsigned TargetClassArrayElementSize = 14; +const unsigned TargetClassVtable = 128; + +const unsigned TargetFieldOffset = 12; + # elif (TARGET_BYTES_PER_WORD == 4) # define TARGET_VW(v) TARGET_V4(v) + typedef uint32_t target_uintptr_t; typedef int32_t target_intptr_t; + const unsigned TargetBytesPerWord = 4; + +const unsigned TargetThreadTailAddress = 2172; +const unsigned TargetThreadStackLimit = 2204; +const unsigned TargetThreadStack = 2148; +const unsigned TargetThreadIp = 2144; +const unsigned TargetThreadVirtualCallTarget = 2176; +const unsigned TargetThreadVirtualCallIndex = 2180; + +const unsigned TargetClassFixedSize = 8; +const unsigned TargetClassArrayElementSize = 10; +const unsigned TargetClassVtable = 68; + +const unsigned TargetFieldOffset = 8; + # else # error # endif #else -typedef uintptr_t target_uintptr_t; -typedef intptr_t target_intptr_t; -const unsigned TargetBytesPerWord = BytesPerWord; +# error #endif const unsigned TargetBitsPerWord = TargetBytesPerWord * 8; +const uintptr_t TargetPointerMask += ((~static_cast(0)) / TargetBytesPerWord) + * TargetBytesPerWord; + +const unsigned TargetArrayLength = TargetBytesPerWord; +const unsigned TargetArrayBody = TargetBytesPerWord * 2; + } // namespace vm #endif//TARGET_H diff --git a/src/type-generator.cpp b/src/type-generator.cpp index eb0141f215..52eef613b6 100644 --- a/src/type-generator.cpp +++ b/src/type-generator.cpp @@ -2116,6 +2116,9 @@ writeMap(Output* out, Object* type) case Object::Scalar: { out->write("Type_"); out->write(memberTypeEnumName(m)); + if (memberNoGC(m)) { + out->write("_nogc"); + } } break; case Object::Array: { diff --git a/src/vector.h b/src/vector.h index a83aed9a7c..6714573b0d 100644 --- a/src/vector.h +++ b/src/vector.h @@ -12,6 +12,7 @@ #define VECTOR_H #include "system.h" +#include "target.h" namespace vm { @@ -103,6 +104,10 @@ class Vector { append(&v, 4); } + void appendTargetAddress(target_uintptr_t v) { + append(&v, TargetBytesPerWord); + } + void appendAddress(uintptr_t v) { append(&v, BytesPerWord); } diff --git a/src/x86.cpp b/src/x86.cpp index 5a90ede640..a1c1a833b5 100644 --- a/src/x86.cpp +++ b/src/x86.cpp @@ -74,13 +74,13 @@ const unsigned StackAlignmentInBytes = 16; const unsigned StackAlignmentInWords = StackAlignmentInBytes / TargetBytesPerWord; bool -isInt8(intptr_t v) +isInt8(target_intptr_t v) { return v == static_cast(v); } bool -isInt32(intptr_t v) +isInt32(target_intptr_t v) { return v == static_cast(v); } @@ -958,11 +958,11 @@ moveCR2(Context* c, UNUSED unsigned aSize, Assembler::Constant* a, maybeRex(c, TargetBytesPerWord, b); opcode(c, 0xb8 + regCode(b)); if (a->value->resolved()) { - c->code.appendAddress(a->value->value()); + c->code.appendTargetAddress(a->value->value()); } else { appendImmediateTask (c, a->value, offset(c), TargetBytesPerWord, promiseOffset); - c->code.appendAddress(static_cast(0)); + c->code.appendTargetAddress(static_cast(0)); } } } @@ -3446,7 +3446,7 @@ class MyAssembler: public Assembler { RUNTIME_ARRAY_BODY(arguments)[i].size, RUNTIME_ARRAY_BODY(arguments)[i].type, RUNTIME_ARRAY_BODY(arguments)[i].operand, - pad(RUNTIME_ARRAY_BODY(arguments)[i].size), + pad(RUNTIME_ARRAY_BODY(arguments)[i].size, TargetBytesPerWord), RegisterOperand, &dst); } else { @@ -3455,7 +3455,7 @@ class MyAssembler: public Assembler { RUNTIME_ARRAY_BODY(arguments)[i].size, RUNTIME_ARRAY_BODY(arguments)[i].type, RUNTIME_ARRAY_BODY(arguments)[i].operand, - pad(RUNTIME_ARRAY_BODY(arguments)[i].size), + pad(RUNTIME_ARRAY_BODY(arguments)[i].size, TargetBytesPerWord), MemoryOperand, &dst); offset += ceiling