move Kotlin/Native metadata deserialization and KLIB reader to a sepa…

…rate module (JetBrains#1908)

1. move Kotlin/Native metadata deserialization and KLIB reader to a separate module
2. decouple KLIB reader from ModuleDescriptor creation

backend.native/build.gradle

@@ -140,11 +140,12 @@ dependencies {
     compilerCompile "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
     compilerCompile "org.jetbrains.kotlin:konan.descriptors:$konanVersion"
     compilerCompile "org.jetbrains.kotlin:konan.metadata:$konanVersion"
-    compilerCompile "org.jetbrains.kotlin:konan.metadata.serializer:$konanVersion"
+    compilerCompile "org.jetbrains.kotlin:konan.serializer:$konanVersion"
 
     cli_bcCompile kotlinCompilerModule
     cli_bcCompile "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
     cli_bcCompile sourceSets.compiler.output
+    cli_bcCompile "org.jetbrains.kotlin:konan.serializer:$konanVersion"
 
     bc_frontendCompile kotlinCompilerModule
 

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -20,7 +20,6 @@ import com.intellij.openapi.Disposable
 import org.jetbrains.annotations.NotNull
 import org.jetbrains.annotations.Nullable
 import org.jetbrains.kotlin.backend.konan.*
-import org.jetbrains.kotlin.backend.konan.serialization.KonanMetadataVersion
 import org.jetbrains.kotlin.cli.common.CLICompiler
 import org.jetbrains.kotlin.cli.common.CLITool
 import org.jetbrains.kotlin.cli.common.CommonCompilerPerformanceManager
@@ -32,14 +31,17 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.WARNING
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.cli.jvm.plugins.PluginCliParser
-import org.jetbrains.kotlin.config.*
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.konan.KonanVersion
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.util.profile
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.utils.KotlinPaths
+import org.jetbrains.kotlin.serialization.konan.KonanMetadataVersion
 
 private class K2NativeCompilerPerformanceManager: CommonCompilerPerformanceManager("Kotlin to Native Compiler")
 class K2Native : CLICompiler<K2NativeCompilerArguments>() {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/InteropUtils.kt

@@ -16,14 +16,8 @@
 
 package org.jetbrains.kotlin.backend.konan
 
-import org.jetbrains.kotlin.backend.konan.descriptors.ClassifierAliasingPackageFragmentDescriptor
-import org.jetbrains.kotlin.backend.konan.descriptors.ExportedForwardDeclarationsPackageFragmentDescriptor
-import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader
-import org.jetbrains.kotlin.backend.konan.serialization.KonanPackageFragment
 import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns
 import org.jetbrains.kotlin.descriptors.ClassDescriptor
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.konan.interop.InteropFqNames
 import org.jetbrains.kotlin.incremental.components.NoLookupLocation
 import org.jetbrains.kotlin.name.FqName
@@ -32,25 +26,6 @@ import org.jetbrains.kotlin.resolve.scopes.MemberScope
 import org.jetbrains.kotlin.types.TypeUtils
 import org.jetbrains.kotlin.util.OperatorNameConventions
 
-interface InteropLibrary {
-    fun createSyntheticPackages(
-            module: ModuleDescriptor,
-            konanPackageFragments: List<KonanPackageFragment>
-    ): List<PackageFragmentDescriptor>
-}
-
-fun createInteropLibrary(reader: KonanLibraryReader): InteropLibrary? {
-    if (reader.manifestProperties.getProperty("interop") != "true") return null
-    val pkg = reader.manifestProperties.getProperty("package") 
-        ?: error("Inconsistent manifest: interop library ${reader.libraryName} should have `package` specified")
-    val exportForwardDeclarations = reader.manifestProperties
-            .getProperty("exportForwardDeclarations").split(' ')
-            .map { it.trim() }.filter { it.isNotEmpty() }
-            .map { FqName(it) }
-
-    return InteropLibraryImpl(FqName(pkg), exportForwardDeclarations)
-}
-
 internal class InteropBuiltIns(builtIns: KonanBuiltIns, vararg konanPrimitives: ClassDescriptor) {
 
     val packageScope = builtIns.builtInsModule.getPackage(InteropFqNames.packageName).memberScope
@@ -183,29 +158,3 @@ private fun MemberScope.getContributedClass(name: String): ClassDescriptor =
 
 private fun MemberScope.getContributedFunctions(name: String) =
         this.getContributedFunctions(Name.identifier(name), NoLookupLocation.FROM_BUILTINS)
-
-private class InteropLibraryImpl(
-        private val packageFqName: FqName,
-        private val exportForwardDeclarations: List<FqName>
-) : InteropLibrary {
-    override fun createSyntheticPackages(
-            module: ModuleDescriptor,
-            konanPackageFragments: List<KonanPackageFragment>
-    ): List<PackageFragmentDescriptor> {
-        val interopPackageFragments = konanPackageFragments.filter { it.fqName == packageFqName }
-
-        val result = mutableListOf<PackageFragmentDescriptor>()
-
-        // Allow references to forwarding declarations to be resolved into classifiers declared in this library:
-        listOf(InteropFqNames.cNamesStructs, InteropFqNames.objCNamesClasses, InteropFqNames.objCNamesProtocols).mapTo(result) { fqName ->
-            ClassifierAliasingPackageFragmentDescriptor(interopPackageFragments, module, fqName)
-        }
-        // TODO: use separate namespaces for structs, enums, Objective-C protocols etc.
-
-        result.add(ExportedForwardDeclarationsPackageFragmentDescriptor(
-                module, packageFqName, exportForwardDeclarations
-        ))
-
-        return result
-    }
-}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -17,9 +17,6 @@
 package org.jetbrains.kotlin.backend.konan
 
 import com.intellij.openapi.project.Project
-import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader
-import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
-import org.jetbrains.kotlin.backend.konan.library.impl.purgeUnneeded
 import org.jetbrains.kotlin.backend.konan.library.resolveImmediateLibraries
 import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive
 import org.jetbrains.kotlin.backend.konan.library.withResolvedDependencies
@@ -35,9 +32,11 @@ import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.descriptors.konan.interop.createForwardDeclarationsModule
 import org.jetbrains.kotlin.konan.TempFiles
 import org.jetbrains.kotlin.konan.file.File
+import org.jetbrains.kotlin.konan.library.KonanLibraryReader
 import org.jetbrains.kotlin.konan.library.defaultResolver
 import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.konan.util.profile
+import org.jetbrains.kotlin.serialization.konan.DefaultKonanModuleDescriptorFactory
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.storage.StorageManager
 
@@ -94,7 +93,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
     private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES)
     private val resolver = defaultResolver(repositories, target, distribution)
 
-    internal val immediateLibraries: List<LibraryReaderImpl> by lazy {
+    internal val immediateLibraries: List<KonanLibraryReader> by lazy {
         val result = resolver.resolveImmediateLibraries(libraryNames, target,
                 currentAbiVersion,
                 configuration.getBoolean(KonanConfigKeys.NOSTDLIB),
@@ -136,7 +135,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         for (klib in libraries) {
             profile("Loading ${klib.libraryName}") {
                 // MutableModuleContext needs ModuleDescriptorImpl, rather than ModuleDescriptor.
-                val moduleDescriptor = klib.moduleDescriptor(specifics)
+                val moduleDescriptor = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(klib, specifics)
                 allMetadata.add(moduleDescriptor)
                 friendLibsSet?.apply {
                     if (contains(klib.libraryFile))
@@ -175,3 +174,6 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
 
 fun CompilerConfiguration.report(priority: CompilerMessageSeverity, message: String) 
     = this.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(priority, message)
+
+private fun <T: KonanLibraryReader> List<T>.purgeUnneeded(config: KonanConfig): List<T> =
+        this.filter{ (!it.isDefaultLibrary && !config.purgeUserLibs) || it.isNeededForLink }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.backend.konan.library
 
 import llvm.LLVMModuleRef
+import org.jetbrains.kotlin.konan.library.KonanLibraryReader
 
 interface KonanLibraryWriter {
     fun addLinkData(linkData: LinkData)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolverExtensions.kt

@@ -16,22 +16,29 @@
 
 package org.jetbrains.kotlin.backend.konan.library
 
-import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
 import org.jetbrains.kotlin.konan.file.File
+import org.jetbrains.kotlin.konan.library.KonanLibraryReader
 import org.jetbrains.kotlin.konan.library.SearchPathResolver
+import org.jetbrains.kotlin.konan.library.impl.LibraryReaderImpl
 import org.jetbrains.kotlin.konan.target.KonanTarget
 
+const val KONAN_CURRENT_ABI_VERSION = 1
+
+/**
+ * Returns the list of [KonanLibraryReader]s given the list of user provided [libraryNames] along with
+ * other parameters: [target], [abiVersion], [noStdLib], [noDefaultLibs].
+ */
 fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List<String>,
                                                  target: KonanTarget,
-                                                 abiVersion: Int = 1,
+                                                 abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
                                                  noStdLib: Boolean = false,
                                                  noDefaultLibs: Boolean = false,
-                                                 logger: ((String) -> Unit)?): List<LibraryReaderImpl> {
+                                                 logger: ((String) -> Unit)?): List<KonanLibraryReader> {
     val userProvidedLibraries = libraryNames
             .map { resolve(it) }
             .map{ LibraryReaderImpl(it, abiVersion, target) }
 
-    val defaultLibraries = defaultLinks(nostdlib = noStdLib, noDefaultLibs = noDefaultLibs).map {
+    val defaultLibraries = defaultLinks(noStdLib = noStdLib, noDefaultLibs = noDefaultLibs).map {
         LibraryReaderImpl(it, abiVersion, target, isDefaultLibrary = true)
     }
 
@@ -44,8 +51,7 @@ fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List<String>,
     return resolvedLibraries.distinctBy { it.libraryFile.absolutePath }
 }
 
-private fun warnOnLibraryDuplicates(resolvedLibraries: List<File>, 
-    logger: ((String) -> Unit)? ) {
+private fun warnOnLibraryDuplicates(resolvedLibraries: List<File>, logger: ((String) -> Unit)? ) {
 
     if (logger == null) return
 
@@ -56,38 +62,46 @@ private fun warnOnLibraryDuplicates(resolvedLibraries: List<File>,
     }
 }
 
-fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<LibraryReaderImpl>,
+/**
+ * For each of the given [immediateLibraries] fills in `resolvedDependencies` field with the
+ * [KonanLibraryReader]s the library !!directly!! depends on.
+ */
+fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<KonanLibraryReader>,
                                                  target: KonanTarget,
                                                  abiVersion: Int) {
-    val cache = mutableMapOf<File, LibraryReaderImpl>()
+    val cache = mutableMapOf<File, KonanLibraryReader>()
     cache.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it })
     var newDependencies = cache.values.toList()
     do {
-        newDependencies = newDependencies.map { library: LibraryReaderImpl ->
+        newDependencies = newDependencies.map { library: KonanLibraryReader ->
             library.unresolvedDependencies
                     .map { resolve(it).absoluteFile }
-                    .map { 
+                    .mapNotNull {
                         if (it in cache) {
                             library.resolvedDependencies.add(cache[it]!!)
                             null
                         } else {
                             val reader = LibraryReaderImpl(it, abiVersion, target)
-                            cache.put(it,reader)
-                            library.resolvedDependencies.add(reader) 
+                            cache[it] = reader
+                            library.resolvedDependencies.add(reader)
                             reader
                         }
-            }.filterNotNull()
+            }
         } .flatten()
     } while (newDependencies.isNotEmpty())
 }
 
-fun List<LibraryReaderImpl>.withResolvedDependencies(): List<LibraryReaderImpl> {
-    val result = mutableSetOf<LibraryReaderImpl>()
+/**
+ * For the given list of [KonanLibraryReader]s returns the list of [KonanLibraryReader]s
+ * that includes the same libraries plus all their (transitive) dependencies.
+ */
+fun List<KonanLibraryReader>.withResolvedDependencies(): List<KonanLibraryReader> {
+    val result = mutableSetOf<KonanLibraryReader>()
     result.addAll(this)
     var newDependencies = result.toList()
     do {
         newDependencies = newDependencies
-            .map { it -> it.resolvedDependencies } .flatten()
+            .map { it -> it.resolvedDependencies }.flatten()
             .filter { it !in result }
         result.addAll(newDependencies)
     } while (newDependencies.isNotEmpty())
@@ -96,9 +110,9 @@ fun List<LibraryReaderImpl>.withResolvedDependencies(): List<LibraryReaderImpl>
 
 fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List<String>,
                                                  target: KonanTarget,
-                                                 abiVersion: Int = 1,
+                                                 abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
                                                  noStdLib: Boolean = false,
-                                                 noDefaultLibs: Boolean = false): List<LibraryReaderImpl> {
+                                                 noDefaultLibs: Boolean = false): List<KonanLibraryReader> {
     val immediateLibraries = resolveImmediateLibraries(
                     libraryNames = libraryNames,
                     target = target,