Improve code related to object layout (JetBrains#2446)

Interop/Runtime/src/native/kotlin/kotlinx/cinterop/Pinning.kt

@@ -44,16 +44,16 @@ inline fun <T : Any, R> T.usePinned(block: (Pinned<T>) -> R): R {
     }
 }
 
-fun Pinned<ByteArray>.addressOf(index: Int): CPointer<ByteVar> = this.addressOfElement(index)
+fun Pinned<ByteArray>.addressOf(index: Int): CPointer<ByteVar> = this.get().addressOfElement(index)
 fun ByteArray.refTo(index: Int): CValuesRef<ByteVar> = this.usingPinned { addressOf(index) }
 
-fun Pinned<ShortArray>.addressOf(index: Int): CPointer<ShortVar> = this.addressOfElement(index)
+fun Pinned<ShortArray>.addressOf(index: Int): CPointer<ShortVar> = this.get().addressOfElement(index)
 fun ShortArray.refTo(index: Int): CValuesRef<ShortVar> = this.usingPinned { addressOf(index) }
 
-fun Pinned<IntArray>.addressOf(index: Int): CPointer<IntVar> = this.addressOfElement(index)
+fun Pinned<IntArray>.addressOf(index: Int): CPointer<IntVar> = this.get().addressOfElement(index)
 fun IntArray.refTo(index: Int): CValuesRef<IntVar> = this.usingPinned { addressOf(index) }
 
-fun Pinned<LongArray>.addressOf(index: Int): CPointer<LongVar> = this.addressOfElement(index)
+fun Pinned<LongArray>.addressOf(index: Int): CPointer<LongVar> = this.get().addressOfElement(index)
 fun LongArray.refTo(index: Int): CValuesRef<LongVar> = this.usingPinned { addressOf(index) }
 
 // TODO: pinning of unsigned arrays involves boxing as they are inline classes wrapping signed arrays.
@@ -69,10 +69,10 @@ fun UIntArray.refTo(index: Int): CValuesRef<UIntVar> = this.usingPinned { addres
 fun Pinned<ULongArray>.addressOf(index: Int): CPointer<ULongVar> = this.get().addressOfElement(index)
 fun ULongArray.refTo(index: Int): CValuesRef<ULongVar> = this.usingPinned { addressOf(index) }
 
-fun Pinned<FloatArray>.addressOf(index: Int): CPointer<FloatVar> = this.addressOfElement(index)
+fun Pinned<FloatArray>.addressOf(index: Int): CPointer<FloatVar> = this.get().addressOfElement(index)
 fun FloatArray.refTo(index: Int): CValuesRef<FloatVar> = this.usingPinned { addressOf(index) }
 
-fun Pinned<DoubleArray>.addressOf(index: Int): CPointer<DoubleVar> = this.addressOfElement(index)
+fun Pinned<DoubleArray>.addressOf(index: Int): CPointer<DoubleVar> = this.get().addressOfElement(index)
 fun DoubleArray.refTo(index: Int): CValuesRef<DoubleVar> = this.usingPinned { addressOf(index) }
 
 private inline fun <T : Any, P : CPointed> T.usingPinned(
@@ -86,21 +86,32 @@ private inline fun <T : Any, P : CPointed> T.usingPinned(
     }
 }
 
-@SymbolName("Kotlin_Arrays_getAddressOfElement")
-private external fun getAddressOfElement(array: Any, index: Int): COpaquePointer
+@SymbolName("Kotlin_Arrays_getByteArrayAddressOfElement")
+private external fun ByteArray.addressOfElement(index: Int): CPointer<ByteVar>
 
-@Suppress("NOTHING_TO_INLINE")
-private inline fun <P : CVariable> Pinned<*>.addressOfElement(index: Int): CPointer<P> =
-        getAddressOfElement(this.get(), index).reinterpret()
+@SymbolName("Kotlin_Arrays_getShortArrayAddressOfElement")
+private external fun ShortArray.addressOfElement(index: Int): CPointer<ShortVar>
 
-@SymbolName("Kotlin_Arrays_getAddressOfElement")
+@SymbolName("Kotlin_Arrays_getIntArrayAddressOfElement")
+private external fun IntArray.addressOfElement(index: Int): CPointer<IntVar>
+
+@SymbolName("Kotlin_Arrays_getLongArrayAddressOfElement")
+private external fun LongArray.addressOfElement(index: Int): CPointer<LongVar>
+
+@SymbolName("Kotlin_Arrays_getByteArrayAddressOfElement")
 private external fun UByteArray.addressOfElement(index: Int): CPointer<UByteVar>
 
-@SymbolName("Kotlin_Arrays_getAddressOfElement")
+@SymbolName("Kotlin_Arrays_getShortArrayAddressOfElement")
 private external fun UShortArray.addressOfElement(index: Int): CPointer<UShortVar>
 
-@SymbolName("Kotlin_Arrays_getAddressOfElement")
+@SymbolName("Kotlin_Arrays_getIntArrayAddressOfElement")
 private external fun UIntArray.addressOfElement(index: Int): CPointer<UIntVar>
 
-@SymbolName("Kotlin_Arrays_getAddressOfElement")
+@SymbolName("Kotlin_Arrays_getLongArrayAddressOfElement")
 private external fun ULongArray.addressOfElement(index: Int): CPointer<ULongVar>
+
+@SymbolName("Kotlin_Arrays_getFloatArrayAddressOfElement")
+private external fun FloatArray.addressOfElement(index: Int): CPointer<FloatVar>
+
+@SymbolName("Kotlin_Arrays_getDoubleArrayAddressOfElement")
+private external fun DoubleArray.addressOfElement(index: Int): CPointer<DoubleVar>

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -1592,7 +1592,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
             functionGenerationContext.gep(objCPtr, bodyOffset)
         } else {
-            LLVMBuildGEP(functionGenerationContext.builder, objectPtr, cValuesOf(kImmOne), 1, "")!!
+            objectPtr
         }
     }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt

@@ -133,7 +133,8 @@ private fun Context.getDeclaredFields(classDescriptor: ClassDescriptor): List<Ir
 }
 
 private fun ContextUtils.createClassBodyType(name: String, fields: List<IrField>): LLVMTypeRef {
-    val fieldTypes = fields.map { getLLVMType(it.type) }
+    val fieldTypes = listOf(runtime.objHeaderType) + fields.map { getLLVMType(it.type) }
+    // TODO: consider adding synthetic ObjHeader field to Any.
 
     val classType = LLVMStructCreateNamed(LLVMGetModuleContext(context.llvmModule), name)!!
 
@@ -369,7 +370,7 @@ private class DeclarationsGeneratorVisitor(override val context: Context) :
                 error(containingClass.descriptor.toString())
             val allFields = classDeclarations.fields
             this.fields[descriptor] = FieldLlvmDeclarations(
-                    allFields.indexOf(descriptor),
+                    allFields.indexOf(descriptor) + 1, // First field is ObjHeader.
                     classDeclarations.bodyType
             )
         } else {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt

@@ -289,9 +289,9 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
                     NullPointer(int32Type), NullPointer(int8Type), NullPointer(kInt8Ptr))
         } else {
             data class FieldRecord(val offset: Int, val type: Int, val name: String)
-            val fields = getStructElements(bodyType).mapIndexedNotNull { index, type ->
+            val fields = getStructElements(bodyType).drop(1).mapIndexedNotNull { index, type ->
                 FieldRecord(
-                        LLVMOffsetOfElement(llvmTargetData, bodyType, index).toInt(),
+                        LLVMOffsetOfElement(llvmTargetData, bodyType, index + 1).toInt(),
                         mapRuntimeType(type),
                         llvmDeclarations.fields[index].name.asString())
             }
@@ -316,7 +316,7 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
     ): ConstPointer {
         assert(descriptor.isInterface)
 
-        val size = 0
+        val size = LLVMStoreSizeOfType(llvmTargetData, kObjHeader).toInt()
 
         val superClass = context.ir.symbols.any.owner
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt

@@ -71,24 +71,20 @@ internal fun StaticData.createConstKotlinArray(arrayClass: IrClass, elements: Li
     return createRef(objHeaderPtr)
 }
 
-internal fun StaticData.createConstKotlinObject(type: IrClass, body: ConstValue): ConstPointer {
+internal fun StaticData.createConstKotlinObject(type: IrClass, vararg fields: ConstValue): ConstPointer {
     val typeInfo = type.typeInfoPtr
-
-    val compositeType = structType(runtime.objHeaderType, body.llvmType)
-
-    val global = this.createGlobal(compositeType, "")
-
-    val objHeaderPtr = global.pointer.getElementPtr(0)
     val objHeader = objHeader(typeInfo)
 
-    global.setInitializer(Struct(compositeType, objHeader, body))
+    val global = this.placeGlobal("", Struct(objHeader, *fields))
     global.setConstant(true)
 
+    val objHeaderPtr = global.pointer.getElementPtr(0)
+
     return createRef(objHeaderPtr)
 }
 
-internal fun StaticData.createInitializer(type: IrClass, body: ConstValue): ConstValue =
-        Struct(objHeader(type.typeInfoPtr), body)
+internal fun StaticData.createInitializer(type: IrClass, vararg fields: ConstValue): ConstValue =
+        Struct(objHeader(type.typeInfoPtr), *fields)
 
 private fun StaticData.getArrayListClass(): ClassDescriptor {
     val module = context.irModule!!.descriptor
@@ -122,15 +118,13 @@ internal fun StaticData.createConstArrayList(array: ConstPointer, length: Int):
         val fqName = it.fqNameSafe.asString()
         sorted.put(fqName, arrayListFields[fqName]!!)
     }
-
-    val body = Struct(*(sorted.values.toTypedArray()))
 
-    return createConstKotlinObject(arrayListClass, body)
+    return createConstKotlinObject(arrayListClass, *sorted.values.toTypedArray())
 }
 
 internal fun StaticData.createUniqueInstance(
         kind: UniqueKind, bodyType: LLVMTypeRef, typeInfo: ConstPointer): ConstPointer {
-    assert (getStructElements(bodyType).isEmpty())
+    assert (getStructElements(bodyType).size == 1) // ObjHeader only.
     val objHeader = when (kind) {
         UniqueKind.UNIT -> objHeader(typeInfo)
         UniqueKind.EMPTY_ARRAY -> arrayHeader(typeInfo, 0)

runtime/src/main/cpp/Arrays.cpp

@@ -489,13 +489,58 @@ KNativePtr Kotlin_ImmutableBlob_asCPointerImpl(KRef thiz, KInt offset) {
   return PrimitiveArrayAddressOfElementAt<KByte>(array, offset);
 }
 
-KNativePtr Kotlin_Arrays_getAddressOfElement(KRef thiz, KInt index) {
+KNativePtr Kotlin_Arrays_getByteArrayAddressOfElement(KRef thiz, KInt index) {
   ArrayHeader* array = thiz->array();
   if (index < 0 || index >= array->count_) {
     ThrowArrayIndexOutOfBoundsException();
   }
 
-  return AddressOfElementAt(array, index);
+  return AddressOfElementAt<KByte>(array, index);
+}
+
+KNativePtr Kotlin_Arrays_getShortArrayAddressOfElement(KRef thiz, KInt index) {
+  ArrayHeader* array = thiz->array();
+  if (index < 0 || index >= array->count_) {
+    ThrowArrayIndexOutOfBoundsException();
+  }
+
+  return AddressOfElementAt<KShort>(array, index);
+}
+
+KNativePtr Kotlin_Arrays_getIntArrayAddressOfElement(KRef thiz, KInt index) {
+  ArrayHeader* array = thiz->array();
+  if (index < 0 || index >= array->count_) {
+    ThrowArrayIndexOutOfBoundsException();
+  }
+
+  return AddressOfElementAt<KInt>(array, index);
+}
+
+KNativePtr Kotlin_Arrays_getLongArrayAddressOfElement(KRef thiz, KInt index) {
+  ArrayHeader* array = thiz->array();
+  if (index < 0 || index >= array->count_) {
+    ThrowArrayIndexOutOfBoundsException();
+  }
+
+  return AddressOfElementAt<KLong>(array, index);
+}
+
+KNativePtr Kotlin_Arrays_getFloatArrayAddressOfElement(KRef thiz, KInt index) {
+  ArrayHeader* array = thiz->array();
+  if (index < 0 || index >= array->count_) {
+    ThrowArrayIndexOutOfBoundsException();
+  }
+
+  return AddressOfElementAt<KFloat>(array, index);
+}
+
+KNativePtr Kotlin_Arrays_getDoubleArrayAddressOfElement(KRef thiz, KInt index) {
+  ArrayHeader* array = thiz->array();
+  if (index < 0 || index >= array->count_) {
+    ThrowArrayIndexOutOfBoundsException();
+  }
+
+  return AddressOfElementAt<KDouble>(array, index);
 }
 
 }  // extern "C"

runtime/src/main/cpp/Atomic.cpp

@@ -23,37 +23,48 @@
 namespace {
 
 struct AtomicReferenceLayout {
+  ObjHeader header;
   KRef value_;
   KInt lock_;
 };
 
+template<typename T> struct AtomicPrimitive {
+  ObjHeader header;
+  volatile T value_;
+};
+
+template <typename T> inline volatile T* getValueLocation(KRef thiz) {
+  AtomicPrimitive<T>* atomic = reinterpret_cast<AtomicPrimitive<T>*>(thiz);
+  return &atomic->value_;
+}
+
 template <typename T> void setImpl(KRef thiz, T value) {
-  volatile T* location = reinterpret_cast<volatile T*>(thiz + 1);
+  volatile T* location = getValueLocation<T>(thiz);
   atomicSet(location, value);
 }
 
 template <typename T> T getImpl(KRef thiz) {
-  volatile T* location = reinterpret_cast<volatile T*>(thiz + 1);
+  volatile T* location = getValueLocation<T>(thiz);
   return atomicGet(location);
 }
 
 template <typename T> T addAndGetImpl(KRef thiz, T delta) {
-  volatile T* location = reinterpret_cast<volatile T*>(thiz + 1);
+  volatile T* location = getValueLocation<T>(thiz);
   return atomicAdd(location, delta);
 }
 
 template <typename T> T compareAndSwapImpl(KRef thiz, T expectedValue, T newValue) {
-    volatile T* location = reinterpret_cast<volatile T*>(thiz + 1);
-    return compareAndSwap(location, expectedValue, newValue);
+  volatile T* location = getValueLocation<T>(thiz);
+  return compareAndSwap(location, expectedValue, newValue);
 }
 
 template <typename T> KBoolean compareAndSetImpl(KRef thiz, T expectedValue, T newValue) {
-    volatile T* location = reinterpret_cast<volatile T*>(thiz + 1);
+    volatile T* location = getValueLocation<T>(thiz);
     return compareAndSet(location, expectedValue, newValue);
 }
 
 inline AtomicReferenceLayout* asAtomicReference(KRef thiz) {
-    return reinterpret_cast<AtomicReferenceLayout*>(thiz + 1);
+    return reinterpret_cast<AtomicReferenceLayout*>(thiz);
 }
 
 }  // namespace
@@ -93,7 +104,7 @@ KLong Kotlin_AtomicLong_compareAndSwap(KRef thiz, KLong expectedValue, KLong new
     // Potentially huge performance penalty, but correct.
     // TODO: reconsider, once target MIPS can do proper 64-bit CAS.
     while (compareAndSwap(&lock64, 0, 1) != 0);
-    KLong* address = reinterpret_cast<KLong*>(thiz + 1);
+    volatile KLong* address = getValueLocation<KLong>(thiz);
     KLong old = *address;
     if (old == expectedValue) {
       *address = newValue;
@@ -111,7 +122,7 @@ KBoolean Kotlin_AtomicLong_compareAndSet(KRef thiz, KLong expectedValue, KLong n
     // TODO: reconsider, once target MIPS can do proper 64-bit CAS.
     KBoolean result = false;
     while (compareAndSwap(&lock64, 0, 1) != 0);
-    KLong* address = reinterpret_cast<KLong*>(thiz + 1);
+    volatile KLong* address = getValueLocation<KLong>(thiz);
     KLong old = *address;
     if (old == expectedValue) {
       result = true;
@@ -129,7 +140,7 @@ void Kotlin_AtomicLong_set(KRef thiz, KLong newValue) {
     // Potentially huge performance penalty, but correct.
     // TODO: reconsider, once target MIPS can do proper 64-bit atomic store.
     while (compareAndSwap(&lock64, 0, 1) != 0);
-    KLong* address = reinterpret_cast<KLong*>(thiz + 1);
+    volatile KLong* address = getValueLocation<KLong>(thiz);
     *address = newValue;
     compareAndSwap(&lock64, 1, 0);
 #else
@@ -142,7 +153,7 @@ KLong Kotlin_AtomicLong_get(KRef thiz) {
     // Potentially huge performance penalty, but correct.
     // TODO: reconsider, once target MIPS can do proper 64-bit atomic store.
     while (compareAndSwap(&lock64, 0, 1) != 0);
-    KLong* address = reinterpret_cast<KLong*>(thiz + 1);
+    volatile KLong* address = getValueLocation<KLong>(thiz);
     KLong value = *address;
     compareAndSwap(&lock64, 1, 0);
     return value;

runtime/src/main/cpp/KDebug.cpp

@@ -45,6 +45,19 @@ constexpr int runtimeTypeSize[] = {
     1                    // BOOLEAN
 };
 
+constexpr int runtimeTypeAlignment[] = {
+    -1,                  // INVALID
+    alignof(ObjHeader*), // OBJECT
+    alignof(int8_t),     // INT8
+    alignof(int16_t),    // INT16
+    alignof(int32_t),    // INT32
+    alignof(int64_t),    // INT64
+    alignof(float),      // FLOAT32
+    alignof(double),     // FLOAT64
+    alignof(void*),      // NATIVE_PTR
+    1                    // BOOLEAN
+};
+
 }  // namespace
 
 extern "C" {
@@ -130,13 +143,16 @@ RUNTIME_USED void* Konan_DebugGetFieldAddress(KRef obj, int index) {
      if (index > obj->array()->count_)
         return nullptr;
 
-      return reinterpret_cast<uint8_t*>(obj->array() + 1) + index * runtimeTypeSize[-extendedTypeInfo->fieldsCount_];
+      int32_t typeIndex = -extendedTypeInfo->fieldsCount_;
+      return reinterpret_cast<uint8_t*>(obj->array())
+          + alignUp(sizeof(struct ArrayHeader), runtimeTypeAlignment[typeIndex])
+          + index * runtimeTypeSize[typeIndex];
    }
 
    if (index >= extendedTypeInfo->fieldsCount_)
      return nullptr;
 
-   return reinterpret_cast<uint8_t*>(obj + 1) + extendedTypeInfo->fieldOffsets_[index];
+   return reinterpret_cast<uint8_t*>(obj) + extendedTypeInfo->fieldOffsets_[index];
 }
 
 // Compute address of field or an array element at the index, or null, if incorrect.