From 4c4c03cea69e99fd28e077aecb380b3596777d11 Mon Sep 17 00:00:00 2001 From: Roman Elizarov Date: Mon, 1 Jun 2020 14:46:41 +0300 Subject: [PATCH] Kotlin/Native multithreading * Provides newSingleThreadedContext. * Provides Dispatchers.Main on iOS, Dispatchers.Default everywhere. * Coroutine references (Job), all kinds of channels and StateFlow are shareable across workers. * Each individual coroutine is confined to a single worker. * Update Dispatchers docs to account for native-mt changes. * Multithreaded support in select expression. * Fix ObjC autorelease object leaks with native-mt dispatchers (#2477) Additional fixes: * Fixed broadcast builder with different thread * Fixed adding a child to a frozen parent job Fixes #462 Fixes #470 Fixes #765 Fixes #1645 Fixes #1751 Fixes #1828 Fixes #1831 Fixes #1764 Fixes #2064 Fixes #2025 Fixes #2226 Fixes #2138 Fixes #2263 Fixes #2322 Fixes #2283 Fixes #2688 Fixes #2398 Fixes #3136 --- README.md | 26 +- gradle.properties | 2 +- kotlin-native-sharing.md | 192 ++++++++++ .../api/kotlinx-coroutines-core.api | 8 +- .../common/src/AbstractCoroutine.kt | 30 +- kotlinx-coroutines-core/common/src/Await.kt | 27 +- .../common/src/Builders.common.kt | 72 +++- .../common/src/CancellableContinuation.kt | 9 +- .../common/src/CancellableContinuationImpl.kt | 54 ++- .../common/src/CoroutineDispatcher.kt | 4 +- .../common/src/CoroutineScope.kt | 2 +- .../common/src/CoroutineStart.kt | 40 -- .../common/src/Dispatchers.common.kt | 14 +- .../common/src/EventLoop.common.kt | 53 +-- .../common/src/Exceptions.common.kt | 2 +- .../common/src/JobSupport.kt | 67 +++- .../common/src/Supervisor.kt | 2 +- kotlinx-coroutines-core/common/src/Timeout.kt | 13 +- .../common/src/channels/AbstractChannel.kt | 115 +++--- .../src/channels/ArrayBroadcastChannel.kt | 31 +- .../src/channels/ArrayBufferState.common.kt | 14 + .../common/src/channels/ArrayChannel.kt | 125 +++---- .../src/channels/ArrayChannelState.common.kt | 12 + .../common/src/channels/Broadcast.kt | 14 +- .../common/src/channels/ConflatedChannel.kt | 44 +-- .../channels/ConflatedChannelState.common.kt | 10 + .../common/src/flow/SharedFlow.kt | 165 ++++----- .../common/src/flow/StateFlow.kt | 29 +- .../src/flow/internal/AbstractSharedFlow.kt | 80 ++-- .../common/src/flow/internal/FlowCoroutine.kt | 2 +- .../common/src/flow/operators/Share.kt | 4 +- .../common/src/internal/Concurrent.common.kt | 7 - .../common/src/internal/DispatchedTask.kt | 19 +- .../src/internal/LockFreeLinkedList.common.kt | 2 +- .../internal/ManualMemoryManagement.common.kt | 8 + .../common/src/internal/Scopes.kt | 20 +- .../common/src/internal/Sharing.common.kt | 30 ++ .../src/internal/Synchronized.common.kt | 2 + .../common/src/internal/ThreadSafeHeap.kt | 1 + .../common/src/intrinsics/Undispatched.kt | 2 + .../common/src/selects/Select.kt | 44 ++- .../common/src/sync/Mutex.kt | 20 +- .../common/test/CoroutineScopeTest.kt | 11 +- .../test/LimitedParallelismSharedTest.kt | 2 + .../test/flow/operators/BackgroundFlowTest.kt | 24 ++ .../common/test/flow/sharing/ShareInTest.kt | 3 + .../concurrent/src/Builders.concurrent.kt | 2 +- .../src/MultithreadedDispatchers.common.kt | 3 - .../src/internal/LockFreeLinkedList.kt | 103 ++++-- .../test/LimitedParallelismConcurrentTest.kt | 61 --- .../concurrent/test/RunBlockingTest.kt | 48 +-- .../ConcurrentExceptionsStressTest.kt | 66 ++++ .../concurrent/test/sync/MutexStressTest.kt | 137 ------- kotlinx-coroutines-core/js/src/Builders.kt | 45 +++ .../js/src/CoroutineContext.kt | 2 +- kotlinx-coroutines-core/js/src/EventLoop.kt | 4 +- kotlinx-coroutines-core/js/src/Exceptions.kt | 3 +- .../js/src/channels/ArrayBufferState.kt | 20 + .../js/src/channels/ArrayChannelState.kt | 23 ++ .../js/src/channels/ConflatedChannelState.kt | 11 + .../js/src/internal/Concurrent.kt | 9 - .../js/src/internal/LinkedList.kt | 2 +- .../js/src/internal/ManualMemoryManagement.kt | 11 + .../js/src/internal/Sharing.kt | 73 ++++ .../js/src/internal/Synchronized.kt | 2 + kotlinx-coroutines-core/jvm/src/Builders.kt | 38 ++ .../jvm/src/CoroutineContext.kt | 16 +- kotlinx-coroutines-core/jvm/src/EventLoop.kt | 4 +- kotlinx-coroutines-core/jvm/src/Exceptions.kt | 5 +- .../jvm/src/ThreadPoolDispatcher.kt | 7 +- .../jvm/src/channels/ArrayBufferState.kt | 23 ++ .../jvm/src/channels/ArrayChannelState.kt | 23 ++ .../jvm/src/channels/ConflatedChannelState.kt | 14 + .../jvm/src/internal/Concurrent.kt | 10 +- .../src/internal/ManualMemoryManagement.kt | 17 + .../jvm/src/internal/Sharing.kt | 94 +++++ .../jvm/src/internal/Synchronized.kt | 2 + .../jvm/test/exceptions/Exceptions.kt | 2 +- .../native/src/Builders.kt | 130 +++++-- .../src/CloseableCoroutineDispatcher.kt | 9 - .../native/src/CoroutineContext.kt | 49 +-- .../native/src/Dispatchers.kt | 61 --- .../native/src/Dispatchers.native.kt | 64 ++++ .../native/src/EventLoop.kt | 70 +++- .../native/src/Exceptions.kt | 11 +- .../native/src/MultithreadedDispatchers.kt | 77 ---- .../native/src/Thread.native.kt | 48 +++ kotlinx-coroutines-core/native/src/Workers.kt | 84 +++++ .../native/src/channels/ArrayBufferState.kt | 28 ++ .../native/src/channels/ArrayChannelState.kt | 33 ++ .../src/channels/ConflatedChannelState.kt | 21 ++ .../native/src/internal/Concurrent.kt | 6 - .../native/src/internal/CopyOnWriteList.kt | 6 +- .../src/internal/ManualMemoryManagement.kt | 21 ++ .../native/src/internal/Sharing.kt | 218 +++++++++++ .../native/src/internal/Synchronized.kt | 3 + .../native/test/DefaultDispatcherTest.kt | 22 ++ .../native/test/EventLoopTest.kt | 55 +++ .../native/test/FreezingTest.kt | 104 ++++++ .../native/test/ParkStressTest.kt | 49 +++ .../native/test/TestBase.kt | 33 +- .../native/test/WithTimeoutNativeTest.kt | 22 ++ .../native/test/WorkerDispatcherTest.kt | 348 ++++++++++++++++++ .../native/test/WorkerTest.kt | 11 +- .../native/test/internal/LinkedListTest.kt | 47 --- .../nativeDarwin/src/Dispatchers.kt | 38 +- .../nativeDarwin/src/EventLoop.kt | 9 + .../nativeDarwin/src/Thread.kt | 73 ++++ .../nativeDarwin/test/AutoreleaseLeakTest.kt | 42 +++ .../nativeDarwin/test/Launcher.kt | 8 +- .../nativeDarwin/test/MainDispatcherTest.kt | 166 ++++----- .../nativeOther/src/Dispatchers.kt | 16 +- .../nativeOther/src/EventLoop.kt | 7 + .../nativeOther/src/Thread.kt | 11 + .../nativeOther/test/Launcher.kt | 3 + .../test/CoroutinesDumpTest.kt | 79 ++-- .../test/DebugProbesTest.kt | 32 +- .../common/src/TestScope.kt | 21 +- .../common/test/RunTestTest.kt | 1 + .../native/test/FailingTests.kt | 3 +- .../test/ObservableSingleTest.kt | 2 +- 121 files changed, 3199 insertions(+), 1279 deletions(-) create mode 100644 kotlin-native-sharing.md create mode 100644 kotlinx-coroutines-core/common/src/channels/ArrayBufferState.common.kt create mode 100644 kotlinx-coroutines-core/common/src/channels/ArrayChannelState.common.kt create mode 100644 kotlinx-coroutines-core/common/src/channels/ConflatedChannelState.common.kt create mode 100644 kotlinx-coroutines-core/common/src/internal/ManualMemoryManagement.common.kt create mode 100644 kotlinx-coroutines-core/common/src/internal/Sharing.common.kt create mode 100644 kotlinx-coroutines-core/common/test/flow/operators/BackgroundFlowTest.kt delete mode 100644 kotlinx-coroutines-core/concurrent/test/LimitedParallelismConcurrentTest.kt create mode 100644 kotlinx-coroutines-core/concurrent/test/exceptions/ConcurrentExceptionsStressTest.kt delete mode 100644 kotlinx-coroutines-core/concurrent/test/sync/MutexStressTest.kt create mode 100644 kotlinx-coroutines-core/js/src/Builders.kt create mode 100644 kotlinx-coroutines-core/js/src/channels/ArrayBufferState.kt create mode 100644 kotlinx-coroutines-core/js/src/channels/ArrayChannelState.kt create mode 100644 kotlinx-coroutines-core/js/src/channels/ConflatedChannelState.kt create mode 100644 kotlinx-coroutines-core/js/src/internal/ManualMemoryManagement.kt create mode 100644 kotlinx-coroutines-core/js/src/internal/Sharing.kt create mode 100644 kotlinx-coroutines-core/jvm/src/channels/ArrayBufferState.kt create mode 100644 kotlinx-coroutines-core/jvm/src/channels/ArrayChannelState.kt create mode 100644 kotlinx-coroutines-core/jvm/src/channels/ConflatedChannelState.kt create mode 100644 kotlinx-coroutines-core/jvm/src/internal/ManualMemoryManagement.kt create mode 100644 kotlinx-coroutines-core/jvm/src/internal/Sharing.kt delete mode 100644 kotlinx-coroutines-core/native/src/CloseableCoroutineDispatcher.kt delete mode 100644 kotlinx-coroutines-core/native/src/Dispatchers.kt create mode 100644 kotlinx-coroutines-core/native/src/Dispatchers.native.kt delete mode 100644 kotlinx-coroutines-core/native/src/MultithreadedDispatchers.kt create mode 100644 kotlinx-coroutines-core/native/src/Thread.native.kt create mode 100644 kotlinx-coroutines-core/native/src/Workers.kt create mode 100644 kotlinx-coroutines-core/native/src/channels/ArrayBufferState.kt create mode 100644 kotlinx-coroutines-core/native/src/channels/ArrayChannelState.kt create mode 100644 kotlinx-coroutines-core/native/src/channels/ConflatedChannelState.kt create mode 100644 kotlinx-coroutines-core/native/src/internal/ManualMemoryManagement.kt create mode 100644 kotlinx-coroutines-core/native/src/internal/Sharing.kt create mode 100644 kotlinx-coroutines-core/native/test/DefaultDispatcherTest.kt create mode 100644 kotlinx-coroutines-core/native/test/EventLoopTest.kt create mode 100644 kotlinx-coroutines-core/native/test/FreezingTest.kt create mode 100644 kotlinx-coroutines-core/native/test/ParkStressTest.kt create mode 100644 kotlinx-coroutines-core/native/test/WithTimeoutNativeTest.kt create mode 100644 kotlinx-coroutines-core/native/test/WorkerDispatcherTest.kt delete mode 100644 kotlinx-coroutines-core/native/test/internal/LinkedListTest.kt create mode 100644 kotlinx-coroutines-core/nativeDarwin/src/EventLoop.kt create mode 100644 kotlinx-coroutines-core/nativeDarwin/src/Thread.kt create mode 100644 kotlinx-coroutines-core/nativeDarwin/test/AutoreleaseLeakTest.kt create mode 100644 kotlinx-coroutines-core/nativeOther/src/EventLoop.kt create mode 100644 kotlinx-coroutines-core/nativeOther/src/Thread.kt diff --git a/README.md b/README.md index 3b24731d25..10fa287370 100644 --- a/README.md +++ b/README.md @@ -184,17 +184,25 @@ Kotlin/JS version of `kotlinx.coroutines` is published as #### Native -Kotlin/Native version of `kotlinx.coroutines` is published as -[`kotlinx-coroutines-core-$platform`](https://mvnrepository.com/search?q=kotlinx-coroutines-core-) where `$platform` is -the target Kotlin/Native platform. [List of currently supported targets](https://github.com/Kotlin/kotlinx.coroutines/blob/master/gradle/compile-native-multiplatform.gradle#L16). +[Kotlin/Native](https://kotlinlang.org/docs/reference/native-overview.html) version of `kotlinx.coroutines` is published as +[`kotlinx-coroutines-core`](https://search.maven.org/artifact/org.jetbrains.kotlinx/kotlinx-coroutines-core/1.4.2/jar) +(follow the link to get the dependency declaration snippet). **Kotlin/Native requires Gradle version 6.0 or later** +to resolve that dependency properly into the corresponding platform-specific artifacts. +Kotlin/Native does not generally provide binary compatibility between versions. +You should use the same version of Kotlin/Native compiler as was used to build `kotlinx.coroutines`. -Only single-threaded code (JS-style) on Kotlin/Native is supported in stable versions. -Additionally, a special `-native-mt` version is released on a regular basis, for the state of multi-threaded coroutines support -please follow the [corresponding issue](https://github.com/Kotlin/kotlinx.coroutines/issues/462) for the additional details. +Kotlin/Native does not support free sharing of mutable objects between threads as on JVM, so several +limitations apply to using coroutines on Kotlin/Native. +See [Sharing and background threads on Kotlin/Native](kotlin-native-sharing.md) for details. -Since Kotlin/Native does not generally provide binary compatibility between versions, -you should use the same version of the Kotlin/Native compiler as was used to build `kotlinx.coroutines`. +Some functions like [newSingleThreadContext] and [runBlocking] are available only for Kotlin/JVM and Kotlin/Native +and are not available on Kotlin/JS. In order to access them from the code that is shared between JVM and Native +you need to enable granular metadata (aka HMPP) in your `gradle.properties` file: + +```properties +kotlin.mpp.enableGranularSourceSetsMetadata=true +``` ## Building and Contributing @@ -224,6 +232,8 @@ See [Contributing Guidelines](CONTRIBUTING.md). [Promise.await]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/await.html [promise]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/promise.html [Window.asCoroutineDispatcher]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/as-coroutine-dispatcher.html +[newSingleThreadContext]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/new-single-thread-context.html +[runBlocking]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/run-blocking.html diff --git a/gradle.properties b/gradle.properties index 63fdf28c62..9e3c6c1670 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,7 +3,7 @@ # # Kotlin -version=1.6.3-SNAPSHOT +version=1.6.3-native-mt-SNAPSHOT group=org.jetbrains.kotlinx kotlin_version=1.6.21 diff --git a/kotlin-native-sharing.md b/kotlin-native-sharing.md new file mode 100644 index 0000000000..ebda9b377d --- /dev/null +++ b/kotlin-native-sharing.md @@ -0,0 +1,192 @@ +# Sharing and background threads on Kotlin/Native + +## Preview disclaimer + +This is a preview release of sharing and backgrounds threads for coroutines on Kotlin/Native. +Details of this implementation will change in the future. See also [Known Problems](#known-problems) +at the end of this document. + +## Introduction + +Kotlin/Native provides an automated memory management that works with mutable data objects separately +and independently in each thread that uses Kotlin/Native runtime. Sharing data between threads is limited: + +* Objects to be shared between threads can be [frozen](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.native.concurrent/freeze.html). + This makes the whole object graph deeply immutable and allows to share it between threads. +* Mutable objects can be wrapped into [DetachedObjectGraph](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.native.concurrent/-detached-object-graph/index.html) + on one thread and later reattached onto the different thread. + +This introduces several differences between Kotlin/JVM and Kotlin/Native in terms of coroutines that must +be accounted for when writing cross-platform applications. + +## Threads and dispatchers + +An active coroutine has a mutable state. It cannot migrate from thread to thread. A coroutine in Kotlin/Native +is always bound to a specific thread. Coroutines that are detached from a thread are currently not supported. + +`kotlinx.coroutines` provides ability to create single-threaded dispatchers for background work +via [newSingleThreadContext] function that is available for both Kotlin/JVM and Kotlin/Native. It is not +recommended shutting down such a dispatcher on Kotlin/Native via [CloseableCoroutineDispatcher.close] function +while the application still working unless you are absolutely sure all coroutines running in this +dispatcher have completed. Unlike Kotlin/JVM, there is no backup default thread that might +execute cleanup code for coroutines that might have been still working in this dispatcher. + +For interoperability with code that is using Kotlin/Native +[Worker](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.native.concurrent/-worker/index.html) +API you can get a reference to single-threaded dispacher's worker using its [CloseableCoroutineDispatcher.worker] property. + +A [Default][Dispatchers.Default] dispatcher on Kotlin/Native contains a single background thread. +This is the dispatcher that is used by default in [GlobalScope]. + +> This limitation may be lifted in the future with the default dispatcher becoming multi-threaded and/or +> its coroutines becoming isolated from each other, so please do not assume that different coroutines running +> in the default dispatcher can share mutable data between themselves. + +A [Main][Dispatchers.Main] dispatcher is +properly defined for all Darwin (Apple) targets, refers to the main thread, and integrates +with Core Foundation main event loop. +On Linux and Windows there is no platform-defined main thread, so [Main][Dispatchers.Main] simply refers +to the current thread that must have been either created with `newSingleThreadContext` or be running +inside [runBlocking] function. + +The main thread of application has two options on using coroutines. +A backend application's main thread shall use [runBlocking]. +A UI application running on one Apple's Darwin OSes shall run +its main queue event loop using `NSRunLoopRun`, `UIApplicationMain`, or ` NSApplicationMain`. +For example, that is how you can have main dispatcher in your own `main` function: + +```kotlin +fun main() { + val mainScope = MainScope() + mainScope.launch { /* coroutine in the main thread */ } + CFRunLoopRun() // run event loop +} +``` + +## Switching threads + +You switch from one dispatcher to another using a regular [withContext] function. For example, a code running +on the main thread might do: + +```kotlin +// in the main thead +val result = withContext(Dispatcher.Default) { + // now executing in background thread +} +// now back to the main thread +result // use result here +``` + +If you capture a reference to any object that is defined in the main thread outside of `withContext` into the +block inside `withContext` then it gets automatically frozen for transfer from the main thread to the +background thread. Freezing is recursive, so you might accidentally freeze unrelated objects that are part of +main thread's mutable state and get +[InvalidMutabilityException](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.native.concurrent/-invalid-mutability-exception/index.html) +later in unrelated parts of your code. +The easiest way to trouble-shoot it is to mark the objects that should not have been frozen using +[ensureNeverFrozen](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.native.concurrent/ensure-never-frozen.html) +function so that you get exception in the very place they were frozen that would pinpoint the corresponding +`withContext` call in your code. + +The `result` of `withContext` call can be used after `withContext` call. It gets automatically frozen +for transfer from background to the main thread, too. + +A disciplined use of threads in Kotlin/Native is to transfer only immutable data between the threads. +Such code works equally well both on Kotlin/JVM and Kotlin/Native. + +> Note: freezing only happens when `withContext` changes from one thread to another. If you call +> `withContext` and execution stays in the same thread, then there is not freezing and mutable data +> can be captured and operated on as usual. + +The same rule on freezing applies to coroutines launched with any builder like [launch], [async], [produce], etc. + +## Communication objects + +All core communication and synchronization objects in `kotlin.coroutines` such as +[Job], [Deferred], [Channel], [BroadcastChannel], [Mutex], and [Semaphore] are _shareable_. +It means that they can be frozen for sharing with another thread and still continue to operate normally. +Any object that is transferred via a frozen (shared) [Deferred] or any [Channel] is also automatically frozen. + +Similar rules apply to [Flow]. When an instance of a [Flow] itself is shared (frozen), then all the references that +are captured in to the lambdas in this flow operators are frozen. Regardless of whether the flow instance itself +was frozen, by default, the whole flow operates in a single thread, so mutable data can freely travel down the +flow from emitter to collector. However, when [flowOn] operator is used to change the thread, then +objects crossing the thread boundary get frozen. + +Note, that if you protect any piece of mutable data with a [Mutex] or a [Semaphore] then it does not +automatically become shareable. In order to share mutable data you have to either +wrap it into [DetachedObjectGraph](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.native.concurrent/-detached-object-graph/index.html) +or use atomic classes ([AtomicInt](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.native.concurrent/-atomic-int/index.html), etc). + +## Cyclic garbage + +Code working in a single thread on Kotlin/Native enjoys fully automatic memory management. Any object graph that +is not referenced anymore is automatically reclaimed even if it contains cyclic chains of references. This does +not extend to shared objects, though. Frozen immutable objects can be freely shared, even if then can contain +reference cycles, but shareable [communication objects](#communication-objects) leak if a reference cycle +to them appears. The easiest way to demonstrate it is to return a reference to a [async] coroutine as its result, +so that the resulting [Deferred] contains a reference to itself: + +```kotlin +// from the main thread call coroutine in a background thread or otherwise share it +val result = GlobalScope.async { + coroutineContext // return its coroutine context that contains a self-reference +} +// now result will not be reclaimed -- memory leak +``` + +A disciplined use of communication objects to transfer immutable data between coroutines does not +result in any memory reclamation problems. + +## Shared channels are resources + +All kinds of [Channel] and [BroadcastChannel] implementations become _resources_ on Kotlin/Native when shared. +They must be closed and fully consumed in order for their memory to be reclaimed. When they are not shared, they +can be dropped in any state and will be reclaimed by memory manager, but a shared channel generally will not be reclaimed +unless closed and consumed. + +This does not affect [Flow], because it is a cold abstraction. Even though [Flow] internally uses channels to transfer +data between threads, it always properly closes these channels when completing collection of data. + +## Known problems + +The current implementation is tested and works for all kinds of single-threaded cases and simple scenarios that +transfer data between two thread like shown in [Switching Threads](#switching-threads) section. However, it is known +to leak memory in scenarios involving concurrency under load, for example when multiple children coroutines running +in different threads are simultaneously cancelled. + + + + +[newSingleThreadContext]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/new-single-thread-context.html +[CloseableCoroutineDispatcher.close]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-closeable-coroutine-dispatcher/close.html +[CloseableCoroutineDispatcher.worker]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-closeable-coroutine-dispatcher/worker.html +[Dispatchers.Default]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-dispatchers/-default.html +[GlobalScope]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-global-scope/index.html +[Dispatchers.Main]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-dispatchers/-main.html +[runBlocking]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/run-blocking.html +[withContext]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/with-context.html +[launch]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/launch.html +[async]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/async.html +[Job]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-job/index.html +[Deferred]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-deferred/index.html + + + +[Flow]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines.flow/-flow/index.html +[flowOn]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines.flow/flow-on.html + + + +[produce]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines.channels/produce.html +[Channel]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines.channels/-channel/index.html +[BroadcastChannel]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines.channels/-broadcast-channel/index.html + + + + +[Mutex]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines.sync/-mutex/index.html +[Semaphore]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines.sync/-semaphore/index.html + + + diff --git a/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api b/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api index d227eb879b..23334c3feb 100644 --- a/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api +++ b/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api @@ -60,6 +60,7 @@ public class kotlinx/coroutines/CancellableContinuationImpl : kotlin/coroutines/ public fun getCallerFrame ()Lkotlin/coroutines/jvm/internal/CoroutineStackFrame; public fun getContext ()Lkotlin/coroutines/CoroutineContext; public fun getContinuationCancellationCause (Lkotlinx/coroutines/Job;)Ljava/lang/Throwable; + public synthetic fun getDelegate$kotlinx_coroutines_core ()Lkotlin/coroutines/Continuation; public final fun getResult ()Ljava/lang/Object; public fun getStackTraceElement ()Ljava/lang/StackTraceElement; public fun initCancellability ()V @@ -167,12 +168,12 @@ public abstract class kotlinx/coroutines/CoroutineDispatcher : kotlin/coroutines public abstract fun dispatch (Lkotlin/coroutines/CoroutineContext;Ljava/lang/Runnable;)V public fun dispatchYield (Lkotlin/coroutines/CoroutineContext;Ljava/lang/Runnable;)V public fun get (Lkotlin/coroutines/CoroutineContext$Key;)Lkotlin/coroutines/CoroutineContext$Element; - public final fun interceptContinuation (Lkotlin/coroutines/Continuation;)Lkotlin/coroutines/Continuation; + public fun interceptContinuation (Lkotlin/coroutines/Continuation;)Lkotlin/coroutines/Continuation; public fun isDispatchNeeded (Lkotlin/coroutines/CoroutineContext;)Z public fun limitedParallelism (I)Lkotlinx/coroutines/CoroutineDispatcher; public fun minusKey (Lkotlin/coroutines/CoroutineContext$Key;)Lkotlin/coroutines/CoroutineContext; public final fun plus (Lkotlinx/coroutines/CoroutineDispatcher;)Lkotlinx/coroutines/CoroutineDispatcher; - public final fun releaseInterceptedContinuation (Lkotlin/coroutines/Continuation;)V + public fun releaseInterceptedContinuation (Lkotlin/coroutines/Continuation;)V public fun toString ()Ljava/lang/String; } @@ -237,8 +238,6 @@ public final class kotlinx/coroutines/CoroutineStart : java/lang/Enum { public static final field DEFAULT Lkotlinx/coroutines/CoroutineStart; public static final field LAZY Lkotlinx/coroutines/CoroutineStart; public static final field UNDISPATCHED Lkotlinx/coroutines/CoroutineStart; - public final fun invoke (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)V - public final fun invoke (Lkotlin/jvm/functions/Function2;Ljava/lang/Object;Lkotlin/coroutines/Continuation;)V public final fun isLazy ()Z public static fun valueOf (Ljava/lang/String;)Lkotlinx/coroutines/CoroutineStart; public static fun values ()[Lkotlinx/coroutines/CoroutineStart; @@ -434,7 +433,6 @@ public class kotlinx/coroutines/JobSupport : kotlinx/coroutines/ChildJob, kotlin public final fun getKey ()Lkotlin/coroutines/CoroutineContext$Key; public final fun getOnJoin ()Lkotlinx/coroutines/selects/SelectClause0; protected fun handleJobException (Ljava/lang/Throwable;)Z - protected final fun initParentJob (Lkotlinx/coroutines/Job;)V public final fun invokeOnCompletion (Lkotlin/jvm/functions/Function1;)Lkotlinx/coroutines/DisposableHandle; public final fun invokeOnCompletion (ZZLkotlin/jvm/functions/Function1;)Lkotlinx/coroutines/DisposableHandle; public fun isActive ()Z diff --git a/kotlinx-coroutines-core/common/src/AbstractCoroutine.kt b/kotlinx-coroutines-core/common/src/AbstractCoroutine.kt index 439a9ac7a5..e27a193c9e 100644 --- a/kotlinx-coroutines-core/common/src/AbstractCoroutine.kt +++ b/kotlinx-coroutines-core/common/src/AbstractCoroutine.kt @@ -6,8 +6,11 @@ package kotlinx.coroutines import kotlinx.coroutines.CoroutineStart.* +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.isReuseSupportedInPlatform import kotlinx.coroutines.intrinsics.* import kotlin.coroutines.* +import kotlin.jvm.* /** * Abstract base class for implementation of coroutines in coroutine builders. @@ -35,22 +38,11 @@ import kotlin.coroutines.* */ @InternalCoroutinesApi public abstract class AbstractCoroutine( - parentContext: CoroutineContext, + @JvmField internal val parentContext: CoroutineContext, initParentJob: Boolean, active: Boolean ) : JobSupport(active), Job, Continuation, CoroutineScope { - init { - /* - * Setup parent-child relationship between the parent in the context and the current coroutine. - * It may cause this coroutine to become _cancelling_ if the parent is already cancelled. - * It is dangerous to install parent-child relationship here if the coroutine class - * operates its state from within onCancelled or onCancelling - * (with exceptions for rx integrations that can't have any parent) - */ - if (initParentJob) initParentJob(parentContext[Job]) - } - /** * The context of this coroutine that includes this coroutine as a [Job]. */ @@ -64,6 +56,18 @@ public abstract class AbstractCoroutine( override val isActive: Boolean get() = super.isActive + init { + /* + * Setup parent-child relationship between the parent in the context and the current coroutine. + * It may cause this coroutine to become _cancelling_ if the parent is already cancelled. + * It is dangerous to install parent-child relationship here if the coroutine class + * operates its state from within onCancelled or onCancelling + * (with exceptions for rx integrations that can't have any parent) + */ + // KLUDGE for native late binding + if (initParentJob && isReuseSupportedInPlatform()) initParentJob(parentContext[Job]) + } + /** * This function is invoked once when the job was completed normally with the specified [value], * right before all the waiters for the coroutine's completion are notified. @@ -123,6 +127,6 @@ public abstract class AbstractCoroutine( * * [LAZY] does nothing. */ public fun start(start: CoroutineStart, receiver: R, block: suspend R.() -> T) { - start(block, receiver, this) + startAbstractCoroutine(start, receiver, this, block) } } diff --git a/kotlinx-coroutines-core/common/src/Await.kt b/kotlinx-coroutines-core/common/src/Await.kt index c1669e2554..9c7fef846b 100644 --- a/kotlinx-coroutines-core/common/src/Await.kt +++ b/kotlinx-coroutines-core/common/src/Await.kt @@ -75,12 +75,12 @@ private class AwaitAll(private val deferreds: Array>) { val deferred = deferreds[i] deferred.start() // To properly await lazily started deferreds AwaitAllNode(cont).apply { - handle = deferred.invokeOnCompletion(asHandler) + setHandle(deferred.invokeOnCompletion(asHandler)) } } val disposer = DisposeHandlersOnCancel(nodes) // Step 2: Set disposer to each node - nodes.forEach { it.disposer = disposer } + nodes.forEach { it.setDisposer(disposer) } // Here we know that if any code the nodes complete, it will dispose the rest // Step 3: Now we can check if continuation is complete if (cont.isCompleted) { @@ -93,7 +93,7 @@ private class AwaitAll(private val deferreds: Array>) { private inner class DisposeHandlersOnCancel(private val nodes: Array) : CancelHandler() { fun disposeAll() { - nodes.forEach { it.handle.dispose() } + nodes.forEach { it.disposeHandle() } } override fun invoke(cause: Throwable?) { disposeAll() } @@ -101,13 +101,17 @@ private class AwaitAll(private val deferreds: Array>) { } private inner class AwaitAllNode(private val continuation: CancellableContinuation>) : JobNode() { - lateinit var handle: DisposableHandle - + private val _handle = atomic(null) private val _disposer = atomic(null) - var disposer: DisposeHandlersOnCancel? - get() = _disposer.value - set(value) { _disposer.value = value } - + + fun setHandle(handle: DisposableHandle) { _handle.value = handle } + fun setDisposer(disposer: DisposeHandlersOnCancel) { _disposer.value = disposer } + + fun disposeHandle() { + _handle.value?.dispose() + _handle.value = null + } + override fun invoke(cause: Throwable?) { if (cause != null) { val token = continuation.tryResumeWithException(cause) @@ -115,12 +119,15 @@ private class AwaitAll(private val deferreds: Array>) { continuation.completeResume(token) // volatile read of disposer AFTER continuation is complete // and if disposer was already set (all handlers where already installed, then dispose them all) - disposer?.disposeAll() + _disposer.value?.disposeAll() } } else if (notCompletedCount.decrementAndGet() == 0) { continuation.resume(deferreds.map { it.getCompleted() }) // Note that all deferreds are complete here, so we don't need to dispose their nodes } + // Release all the refs for Kotlin/Native + _handle.value = null + _disposer.value = null } } } diff --git a/kotlinx-coroutines-core/common/src/Builders.common.kt b/kotlinx-coroutines-core/common/src/Builders.common.kt index 3dea68cfde..4966c9b899 100644 --- a/kotlinx-coroutines-core/common/src/Builders.common.kt +++ b/kotlinx-coroutines-core/common/src/Builders.common.kt @@ -108,10 +108,10 @@ private class LazyDeferredCoroutine( parentContext: CoroutineContext, block: suspend CoroutineScope.() -> T ) : DeferredCoroutine(parentContext, active = false) { - private val continuation = block.createCoroutineUnintercepted(this, this) + private val saved = saveLazyCoroutine(this, this, block) override fun onStart() { - continuation.startCoroutineCancellable(this) + startLazyCoroutine(saved, this, this) } } @@ -157,7 +157,7 @@ public suspend fun withContext( newContext.ensureActive() // FAST PATH #1 -- new context is the same as the old one if (newContext === oldContext) { - val coroutine = ScopeCoroutine(newContext, uCont) + val coroutine = ScopeCoroutine(newContext, uCont, true) return@sc coroutine.startUndispatchedOrReturn(coroutine, block) } // FAST PATH #2 -- the new dispatcher is the same as the old one (something else changed) @@ -171,7 +171,7 @@ public suspend fun withContext( } // SLOW PATH -- use new dispatcher val coroutine = DispatchedCoroutine(newContext, uCont) - block.startCoroutineCancellable(coroutine, coroutine) + coroutine.start(CoroutineStart.DEFAULT, coroutine, block) coroutine.getResult() } } @@ -186,6 +186,55 @@ public suspend inline operator fun CoroutineDispatcher.invoke( noinline block: suspend CoroutineScope.() -> T ): T = withContext(this, block) +internal fun startCoroutineImpl( + start: CoroutineStart, + receiver: R, + completion: Continuation, + onCancellation: ((cause: Throwable) -> Unit)?, + block: suspend R.() -> T +) = when (start) { + CoroutineStart.DEFAULT -> block.startCoroutineCancellable(receiver, completion, onCancellation) + CoroutineStart.ATOMIC -> block.startCoroutine(receiver, completion) + CoroutineStart.UNDISPATCHED -> block.startCoroutineUndispatched(receiver, completion) + CoroutineStart.LAZY -> Unit // will start lazily +} + +// --------------- Kotlin/Native specialization hooks --------------- + +// todo: impl a separate startCoroutineCancellable as a fast-path for startCoroutine(DEFAULT, ...) +internal expect fun startCoroutine( + start: CoroutineStart, + receiver: R, + completion: Continuation, + onCancellation: ((cause: Throwable) -> Unit)? = null, + block: suspend R.() -> T +) + +// initParentJob + startCoroutine +internal expect fun startAbstractCoroutine( + start: CoroutineStart, + receiver: R, + coroutine: AbstractCoroutine, + block: suspend R.() -> T +) + +/** + * On JVM & JS lazy coroutines are eagerly started (to record creation trace), the started later. + * On Native the block is saved so that it can be shared with another worker, the created and started later. + */ +internal expect fun saveLazyCoroutine( + coroutine: AbstractCoroutine, + receiver: R, + block: suspend R.() -> T +): Any + +// saved == result of saveLazyCoroutine that was stored in LazyXxxCoroutine class +internal expect fun startLazyCoroutine( + saved: Any, + coroutine: AbstractCoroutine, + receiver: R +) + // --------------- implementation --------------- private open class StandaloneCoroutine( @@ -202,10 +251,9 @@ private class LazyStandaloneCoroutine( parentContext: CoroutineContext, block: suspend CoroutineScope.() -> Unit ) : StandaloneCoroutine(parentContext, active = false) { - private val continuation = block.createCoroutineUnintercepted(this, this) - + private val saved = saveLazyCoroutine(this, this, block) override fun onStart() { - continuation.startCoroutineCancellable(this) + startLazyCoroutine(saved, this, this) } } @@ -223,11 +271,15 @@ private const val RESUMED = 2 internal class DispatchedCoroutine( context: CoroutineContext, uCont: Continuation -) : ScopeCoroutine(context, uCont) { +) : ScopeCoroutine(context, uCont, true) { // this is copy-and-paste of a decision state machine inside AbstractionContinuation // todo: we may some-how abstract it via inline class private val _decision = atomic(UNDECIDED) + protected override fun initParentForNativeUndispatchedCoroutine() { + // Will be inited by AbstractCoroutine.start + } + private fun trySuspend(): Boolean { _decision.loop { decision -> when (decision) { @@ -257,11 +309,13 @@ internal class DispatchedCoroutine( override fun afterResume(state: Any?) { if (tryResume()) return // completed before getResult invocation -- bail out // Resume in a cancellable way because we have to switch back to the original dispatcher - uCont.intercepted().resumeCancellableWith(recoverResult(state, uCont)) + uCont.shareableInterceptedResumeCancellableWith(recoverResult(state, uCont)) } fun getResult(): Any? { if (trySuspend()) return COROUTINE_SUSPENDED + // When scope coroutine does not suspend on Kotlin/Native it shall dispose its continuation which it will not use + disposeContinuation { uCont } // otherwise, onCompletionInternal was already invoked & invoked tryResume, and the result is in the state val state = this.state.unboxState() if (state is CompletedExceptionally) throw state.cause diff --git a/kotlinx-coroutines-core/common/src/CancellableContinuation.kt b/kotlinx-coroutines-core/common/src/CancellableContinuation.kt index 2c2f1b8ff6..5b55935171 100644 --- a/kotlinx-coroutines-core/common/src/CancellableContinuation.kt +++ b/kotlinx-coroutines-core/common/src/CancellableContinuation.kt @@ -336,9 +336,12 @@ internal suspend inline fun suspendCancellableCoroutineReusable( } internal fun getOrCreateCancellableContinuation(delegate: Continuation): CancellableContinuationImpl { - // If used outside of our dispatcher - if (delegate !is DispatchedContinuation) { - return CancellableContinuationImpl(delegate, MODE_CANCELLABLE) + // NOTE: Reuse is not support on Kotlin/Native due to platform peculiarities making it had to properly + // split DispatchedContinuation / CancellableContinuationImpl state across workers. + if (!isReuseSupportedInPlatform() || delegate !is DispatchedContinuation) { + return CancellableContinuationImpl(delegate, MODE_CANCELLABLE).also { + if (delegate is DispatchedContinuation) it.initCancellability() + } } /* * Attempt to claim reusable instance. diff --git a/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt b/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt index 1a0169b65d..08410ccdc4 100644 --- a/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt +++ b/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt @@ -24,13 +24,15 @@ internal val RESUME_TOKEN = Symbol("RESUME_TOKEN") */ @PublishedApi internal open class CancellableContinuationImpl( - final override val delegate: Continuation, + delegate: Continuation, resumeMode: Int ) : DispatchedTask(resumeMode), CancellableContinuation, CoroutineStackFrame { init { assert { resumeMode != MODE_UNINITIALIZED } // invalid mode for CancellableContinuationImpl } + @PublishedApi // for Kotlin/Native + final override val delegate: Continuation = delegate.asShareable() public override val context: CoroutineContext = delegate.context /* @@ -72,7 +74,10 @@ internal open class CancellableContinuationImpl( */ private val _state = atomic(Active) - private var parentHandle: DisposableHandle? = null + private val _parentHandle = atomic(null) + private var parentHandle: DisposableHandle? + get() = _parentHandle.value + set(value) { _parentHandle.value = value } internal val state: Any? get() = _state.value @@ -107,7 +112,8 @@ internal open class CancellableContinuationImpl( } } - private fun isReusable(): Boolean = resumeMode.isReusableMode && (delegate as DispatchedContinuation<*>).isReusable() + // todo: It is never reusable on Kotlin/Native due to architectural peculiarities + private fun isReusable(): Boolean = isReuseSupportedInPlatform() && resumeMode.isReusableMode && (delegate as DispatchedContinuation<*>).isReusable() /** * Resets cancellability state in order to [suspendCancellableCoroutineReusable] to work. @@ -130,7 +136,7 @@ internal open class CancellableContinuationImpl( } public override val callerFrame: CoroutineStackFrame? - get() = delegate as? CoroutineStackFrame + get() = delegate.asLocal() as? CoroutineStackFrame public override fun getStackTraceElement(): StackTraceElement? = null @@ -185,7 +191,8 @@ internal open class CancellableContinuationImpl( } } - internal fun parentCancelled(cause: Throwable) { + internal fun parentCancelled(parentJob: Job) { + val cause = getContinuationCancellationCause(parentJob) if (cancelLater(cause)) return cancel(cause) // Even if cancellation has failed, we should detach child to avoid potential leak @@ -280,6 +287,8 @@ internal open class CancellableContinuationImpl( } return COROUTINE_SUSPENDED } + // When cancellation does not suspend on Kotlin/Native it shall dispose its continuation which it will not use + disposeContinuation { delegate } // otherwise, onCompletionInternal was already invoked & invoked tryResume, and the result is in the state if (isReusable) { // release claimed reusable continuation for the future reuse @@ -514,19 +523,22 @@ internal open class CancellableContinuationImpl( } override fun CoroutineDispatcher.resumeUndispatched(value: T) { - val dc = delegate as? DispatchedContinuation + val dc = delegate.asLocalOrNullIfNotUsed() as? DispatchedContinuation resumeImpl(value, if (dc?.dispatcher === this) MODE_UNDISPATCHED else resumeMode) } override fun CoroutineDispatcher.resumeUndispatchedWithException(exception: Throwable) { - val dc = delegate as? DispatchedContinuation + val dc = delegate.asLocalOrNullIfNotUsed() as? DispatchedContinuation resumeImpl(CompletedExceptionally(exception), if (dc?.dispatcher === this) MODE_UNDISPATCHED else resumeMode) } @Suppress("UNCHECKED_CAST") override fun getSuccessfulResult(state: Any?): T = when (state) { - is CompletedContinuation -> state.result as T + is CompletedContinuation -> { + state.releaseHandlers() + state.result as T + } else -> state as T } @@ -576,17 +588,37 @@ private class InvokeOnCancel( // Clashes with InvokeOnCancellation } // Completed with additional metadata -private data class CompletedContinuation( +private class CompletedContinuation( @JvmField val result: Any?, - @JvmField val cancelHandler: CancelHandler? = null, // installed via invokeOnCancellation - @JvmField val onCancellation: ((cause: Throwable) -> Unit)? = null, // installed via resume block + cancelHandler: CancelHandler? = null, // installed via invokeOnCancellation + onCancellation: ((cause: Throwable) -> Unit)? = null, // installed via resume block @JvmField val idempotentResume: Any? = null, @JvmField val cancelCause: Throwable? = null ) { + private val _cancelHandler = atomic(cancelHandler) + private val _onCancellation = atomic(onCancellation) + + val cancelHandler: CancelHandler? get() = _cancelHandler.value + val onCancellation: ((cause: Throwable) -> Unit)? get() = _onCancellation.value val cancelled: Boolean get() = cancelCause != null + fun releaseHandlers() { + _cancelHandler.value = null + _onCancellation.value = null + } + fun invokeHandlers(cont: CancellableContinuationImpl<*>, cause: Throwable) { cancelHandler?.let { cont.callCancelHandler(it, cause) } onCancellation?.let { cont.callOnCancellation(it, cause) } + releaseHandlers() } + + fun copy( + result: Any? = this.result, + cancelHandler: CancelHandler? = this.cancelHandler, + onCancellation: ((cause: Throwable) -> Unit)? = this.onCancellation, + idempotentResume: Any? = this.idempotentResume, + cancelCause: Throwable? = this.cancelCause + ) = + CompletedContinuation(result, cancelHandler, onCancellation, idempotentResume, cancelCause) } diff --git a/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt b/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt index 71b7ec726f..ce80490b9a 100644 --- a/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt +++ b/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt @@ -136,10 +136,10 @@ public abstract class CoroutineDispatcher : * This method should generally be exception-safe. An exception thrown from this method * may leave the coroutines that use this dispatcher in the inconsistent and hard to debug state. */ - public final override fun interceptContinuation(continuation: Continuation): Continuation = + public override fun interceptContinuation(continuation: Continuation): Continuation = DispatchedContinuation(this, continuation) - public final override fun releaseInterceptedContinuation(continuation: Continuation<*>) { + public override fun releaseInterceptedContinuation(continuation: Continuation<*>) { /* * Unconditional cast is safe here: we only return DispatchedContinuation from `interceptContinuation`, * any ClassCastException can only indicate compiler bug diff --git a/kotlinx-coroutines-core/common/src/CoroutineScope.kt b/kotlinx-coroutines-core/common/src/CoroutineScope.kt index b0928d5c58..a87cef3c5e 100644 --- a/kotlinx-coroutines-core/common/src/CoroutineScope.kt +++ b/kotlinx-coroutines-core/common/src/CoroutineScope.kt @@ -260,7 +260,7 @@ public suspend fun coroutineScope(block: suspend CoroutineScope.() -> R): R callsInPlace(block, InvocationKind.EXACTLY_ONCE) } return suspendCoroutineUninterceptedOrReturn { uCont -> - val coroutine = ScopeCoroutine(uCont.context, uCont) + val coroutine = ScopeCoroutine(uCont.context, uCont, true) coroutine.startUndispatchedOrReturn(coroutine, block) } } diff --git a/kotlinx-coroutines-core/common/src/CoroutineStart.kt b/kotlinx-coroutines-core/common/src/CoroutineStart.kt index 6059829c23..a3a47d894e 100644 --- a/kotlinx-coroutines-core/common/src/CoroutineStart.kt +++ b/kotlinx-coroutines-core/common/src/CoroutineStart.kt @@ -4,8 +4,6 @@ package kotlinx.coroutines import kotlinx.coroutines.CoroutineStart.* -import kotlinx.coroutines.intrinsics.* -import kotlin.coroutines.* /** * Defines start options for coroutines builders. @@ -75,44 +73,6 @@ public enum class CoroutineStart { */ UNDISPATCHED; - /** - * Starts the corresponding block as a coroutine with this coroutine's start strategy. - * - * * [DEFAULT] uses [startCoroutineCancellable]. - * * [ATOMIC] uses [startCoroutine]. - * * [UNDISPATCHED] uses [startCoroutineUndispatched]. - * * [LAZY] does nothing. - * - * @suppress **This an internal API and should not be used from general code.** - */ - @InternalCoroutinesApi - public operator fun invoke(block: suspend () -> T, completion: Continuation): Unit = - when (this) { - DEFAULT -> block.startCoroutineCancellable(completion) - ATOMIC -> block.startCoroutine(completion) - UNDISPATCHED -> block.startCoroutineUndispatched(completion) - LAZY -> Unit // will start lazily - } - - /** - * Starts the corresponding block with receiver as a coroutine with this coroutine start strategy. - * - * * [DEFAULT] uses [startCoroutineCancellable]. - * * [ATOMIC] uses [startCoroutine]. - * * [UNDISPATCHED] uses [startCoroutineUndispatched]. - * * [LAZY] does nothing. - * - * @suppress **This an internal API and should not be used from general code.** - */ - @InternalCoroutinesApi - public operator fun invoke(block: suspend R.() -> T, receiver: R, completion: Continuation): Unit = - when (this) { - DEFAULT -> block.startCoroutineCancellable(receiver, completion) - ATOMIC -> block.startCoroutine(receiver, completion) - UNDISPATCHED -> block.startCoroutineUndispatched(receiver, completion) - LAZY -> Unit // will start lazily - } - /** * Returns `true` when [LAZY]. * diff --git a/kotlinx-coroutines-core/common/src/Dispatchers.common.kt b/kotlinx-coroutines-core/common/src/Dispatchers.common.kt index 28e67a423d..42604e3ffa 100644 --- a/kotlinx-coroutines-core/common/src/Dispatchers.common.kt +++ b/kotlinx-coroutines-core/common/src/Dispatchers.common.kt @@ -15,8 +15,11 @@ public expect object Dispatchers { * [launch][CoroutineScope.launch], [async][CoroutineScope.async], etc * if neither a dispatcher nor any other [ContinuationInterceptor] is specified in their context. * - * It is backed by a shared pool of threads on JVM. By default, the maximum number of threads used - * by this dispatcher is equal to the number of CPU cores, but is at least two. + * Its implementation depends on the platform: + * - On Kotlin/JVM it is backed by a shared pool of threads. By default, the maximum number of threads used + * by this dispatcher is equal to the number of CPU cores, but is at least two. + * - On Kotlin/JS it backed by the JS event loop. + * - On Kotlin/Native it is backed by a single background thread that is created on the first use. */ public val Default: CoroutineDispatcher @@ -30,11 +33,14 @@ public expect object Dispatchers { * - On JS and Native it is equivalent to the [Default] dispatcher. * - On JVM it is either the Android main thread dispatcher, JavaFx or Swing EDT dispatcher. It is chosen by the * [`ServiceLoader`](https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html). + * - On Kotlin/JS it is equivalent to the [Default] dispatcher. + * - On Kotlin/Native Apple platforms it maps to the Darwin main thread. + * - On other Kotlin/Native platforms it is equivalent to the [Default] dispatcher. * - * In order to work with the `Main` dispatcher, the following artifact should be added to the project runtime dependencies: + * In order to work with the `Main` dispatcher on Kotlin/JVM, the following artifact should be added to the project runtime dependencies: * - `kotlinx-coroutines-android` — for Android Main thread dispatcher * - `kotlinx-coroutines-javafx` — for JavaFx Application thread dispatcher - * - `kotlinx-coroutines-swing` — for Swing EDT dispatcher + * - `kotlinx-coroutines-swing` — for Swing EDT dispatcher. * * Implementation note: [MainCoroutineDispatcher.immediate] is not supported on the Native and JS platforms. */ diff --git a/kotlinx-coroutines-core/common/src/EventLoop.common.kt b/kotlinx-coroutines-core/common/src/EventLoop.common.kt index 12940c54e2..6284453f36 100644 --- a/kotlinx-coroutines-core/common/src/EventLoop.common.kt +++ b/kotlinx-coroutines-core/common/src/EventLoop.common.kt @@ -66,7 +66,9 @@ internal abstract class EventLoop : CoroutineDispatcher() { public fun processUnconfinedEvent(): Boolean { val queue = unconfinedQueue ?: return false val task = queue.removeFirstOrNull() ?: return false - task.run() + platformAutoreleasePool { + task.run() + } return true } /** @@ -235,14 +237,14 @@ internal abstract class EventLoopImplBase: EventLoopImplPlatform(), Delay { val timeNanos = delayToNanos(timeMillis) if (timeNanos < MAX_DELAY_NS) { val now = nanoTime() - DelayedResumeTask(now + timeNanos, continuation).also { task -> + DelayedResumeTask(now + timeNanos, continuation, asShareable()).also { task -> /* - * Order is important here: first we schedule the heap and only then - * publish it to continuation. Otherwise, `DelayedResumeTask` would - * have to know how to be disposed of even when it wasn't scheduled yet. - */ + * Order is important here: first we schedule the heap and only then + * publish it to continuation. Otherwise, `DelayedResumeTask` would + * have to know how to be disposed of even when it wasn't scheduled yet. + */ schedule(now, task) - continuation.disposeOnCancellation(task) + continuation.disposeOnCancellation(task.asShareable()) } } } @@ -253,7 +255,7 @@ internal abstract class EventLoopImplBase: EventLoopImplPlatform(), Delay { val now = nanoTime() DelayedRunnableTask(now + timeNanos, block).also { task -> schedule(now, task) - } + }.asShareable() } else { NonDisposableHandle } @@ -414,7 +416,7 @@ internal abstract class EventLoopImplBase: EventLoopImplPlatform(), Delay { * into heap to avoid overflow and corruption of heap data structure. */ @JvmField var nanoTime: Long - ) : Runnable, Comparable, DisposableHandle, ThreadSafeHeapNode { + ) : ShareableRefHolder(), Runnable, Comparable, DisposableHandle, ThreadSafeHeapNode { @Volatile private var _heap: Any? = null // null | ThreadSafeHeap | DISPOSED_TASK @@ -488,25 +490,31 @@ internal abstract class EventLoopImplBase: EventLoopImplPlatform(), Delay { @Suppress("UNCHECKED_CAST") (heap as? DelayedTaskQueue)?.remove(this) // remove if it is in heap (first) _heap = DISPOSED_TASK // never add again to any heap + disposeSharedRef() } - override fun toString(): String = "Delayed[nanos=$nanoTime]" + override fun toString(): String = "Delayed@$hexAddress[nanos=$nanoTime]" } - private inner class DelayedResumeTask( + private class DelayedResumeTask( nanoTime: Long, - private val cont: CancellableContinuation + private val cont: CancellableContinuation, + private val dispatcher: CoroutineDispatcher ) : DelayedTask(nanoTime) { - override fun run() { with(cont) { resumeUndispatched(Unit) } } - override fun toString(): String = super.toString() + cont.toString() + override fun run() { + disposeSharedRef() + with(cont) { dispatcher.resumeUndispatched(Unit) } + } } private class DelayedRunnableTask( nanoTime: Long, private val block: Runnable ) : DelayedTask(nanoTime) { - override fun run() { block.run() } - override fun toString(): String = super.toString() + block.toString() + override fun run() { + disposeSharedRef() + block.run() + } } /** @@ -531,12 +539,6 @@ internal abstract class EventLoopImplBase: EventLoopImplPlatform(), Delay { internal expect fun createEventLoop(): EventLoop -internal expect fun nanoTime(): Long - -internal expect object DefaultExecutor { - public fun enqueue(task: Runnable) -} - /** * Used by Darwin targets to wrap a [Runnable.run] call in an Objective-C Autorelease Pool. It is a no-op on JVM, JS and * non-Darwin native targets. @@ -547,3 +549,10 @@ internal expect object DefaultExecutor { * pool management, it must manage the pool creation and pool drainage manually. */ internal expect inline fun platformAutoreleasePool(crossinline block: () -> Unit) + +internal expect fun nanoTime(): Long + +internal expect object DefaultExecutor { + public fun enqueue(task: Runnable) +} + diff --git a/kotlinx-coroutines-core/common/src/Exceptions.common.kt b/kotlinx-coroutines-core/common/src/Exceptions.common.kt index 6d5442dfdc..547b0704ce 100644 --- a/kotlinx-coroutines-core/common/src/Exceptions.common.kt +++ b/kotlinx-coroutines-core/common/src/Exceptions.common.kt @@ -22,7 +22,7 @@ internal expect class JobCancellationException( cause: Throwable?, job: Job ) : CancellationException { - internal val job: Job + internal val job: Job? } internal class CoroutinesInternalError(message: String, cause: Throwable) : Error(message, cause) diff --git a/kotlinx-coroutines-core/common/src/JobSupport.kt b/kotlinx-coroutines-core/common/src/JobSupport.kt index 0a3dd23472..c5cfd4010f 100644 --- a/kotlinx-coroutines-core/common/src/JobSupport.kt +++ b/kotlinx-coroutines-core/common/src/JobSupport.kt @@ -7,6 +7,7 @@ package kotlinx.coroutines import kotlinx.atomicfu.* import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.SynchronizedObject import kotlinx.coroutines.intrinsics.* import kotlinx.coroutines.selects.* import kotlin.coroutines.* @@ -139,7 +140,7 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren * Initializes parent job. * It shall be invoked at most once after construction after all other initialization. */ - protected fun initParentJob(parent: Job?) { + internal fun initParentJob(parent: Job?) { assert { parentHandle == null } if (parent == null) { parentHandle = NonDisposableHandle @@ -238,6 +239,7 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren assert { casSuccess } // And process all post-completion actions completeStateFinalization(state, finalState) + disposeLockFreeLinkedList { state.list } // only needed on Kotlin/Native return finalState } @@ -293,6 +295,7 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren onCancelling(null) // simple state is not a failure onCompletionInternal(update) completeStateFinalization(state, update) + disposeLockFreeLinkedList { state as? JobNode } // only needed on Kotlin/Native return true } @@ -498,6 +501,7 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren } } else -> { // is complete + disposeLockFreeLinkedList { node } // :KLUDGE: We have to invoke a handler in platform-specific way via `invokeIt` extension, // because we play type tricks on Kotlin/JS and handler is not necessarily a function there if (invokeImmediately) handler.invokeIt((state as? CompletedExceptionally)?.cause) @@ -527,11 +531,15 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren // try to promote it to LIST state with the corresponding state val list = NodeList() val update = if (state.isActive) list else InactiveNodeList(list) - _state.compareAndSet(state, update) + if (!_state.compareAndSet(state, update)) { + disposeLockFreeLinkedList { list } + } } private fun promoteSingleToNodeList(state: JobNode) { // try to promote it to list (SINGLE+ state) + // Note: on Kotlin/Native we don't have to dispose NodeList() that we've failed to add, since + // it does not have cyclic references when addOneIfEmpty fails. state.addOneIfEmpty(NodeList()) // it must be in SINGLE+ state or state has changed (node could have need removed from state) val list = state.nextNode // either our NodeList or somebody else won the race, updated state @@ -592,7 +600,10 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren is JobNode -> { // SINGE/SINGLE+ state -- one completion handler if (state !== node) return // a different job node --> we were already removed // try remove and revert back to empty state - if (_state.compareAndSet(state, EMPTY_ACTIVE)) return + if (_state.compareAndSet(state, EMPTY_ACTIVE)) { + disposeLockFreeLinkedList { state } + return + } } is Incomplete -> { // may have a list of completion handlers // remove node from the list if there is a list @@ -790,7 +801,11 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren val list = getOrPromoteCancellingList(state) ?: return false // Create cancelling state (with rootCause!) val cancelling = Finishing(list, false, rootCause) - if (!_state.compareAndSet(state, cancelling)) return false + if (!_state.compareAndSet(state, cancelling)) { + // Dispose if the list was just freshly allocated + disposeLockFreeLinkedList { list.takeIf { list !== state.list } } + return false + } // Notify listeners notifyCancelling(list, rootCause) return true @@ -886,7 +901,11 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren // We do it as early is possible while still holding the lock. This ensures that we cancelImpl asap // (if somebody else is faster) and we synchronize all the threads on this finishing lock asap. if (finishing !== state) { - if (!_state.compareAndSet(state, finishing)) return COMPLETING_RETRY + if (!_state.compareAndSet(state, finishing)) { + // Dispose if the list was just freshly allocated + disposeLockFreeLinkedList { list.takeIf { list !== state.list } } + return COMPLETING_RETRY + } } // ## IMPORTANT INVARIANT: Only one thread (that had set isCompleting) can go past this point assert { !finishing.isSealed } // cannot be sealed @@ -1099,15 +1118,20 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren // Seals current state and returns list of exceptions // guarded by `synchronized(this)` fun sealLocked(proposedException: Throwable?): List { - val list = when(val eh = exceptionsHolder) { // volatile read + var list = when(val eh = exceptionsHolder) { // volatile read null -> allocateList() is Throwable -> allocateList().also { it.add(eh) } is ArrayList<*> -> eh as ArrayList else -> error("State is $eh") // already sealed -- cannot happen } val rootCause = this.rootCause // volatile read - rootCause?.let { list.add(0, it) } // note -- rootCause goes to the beginning - if (proposedException != null && proposedException != rootCause) list.add(proposedException) + rootCause?.let { + // note -- rootCause goes to the beginning + list.addOrUpdate(0, it) { list = it } + } + if (proposedException != null && proposedException != rootCause) { + list.addOrUpdate(proposedException) { list = it } + } exceptionsHolder = SEALED return list } @@ -1127,10 +1151,9 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren exceptionsHolder = allocateList().apply { add(eh) add(exception) - } } - is ArrayList<*> -> (eh as ArrayList).add(exception) + is ArrayList<*> -> (eh as ArrayList).addOrUpdate(exception) { exceptionsHolder = it } else -> error("State is $eh") // already sealed -- cannot happen } } @@ -1313,8 +1336,6 @@ private class Empty(override val isActive: Boolean) : Incomplete { } internal open class JobImpl(parent: Job?) : JobSupport(true), CompletableJob { - init { initParentJob(parent) } - override val onCancelComplete get() = true /* * Check whether parent is able to handle exceptions as well. * With this check, an exception in that pattern will be handled once: @@ -1325,18 +1346,24 @@ internal open class JobImpl(parent: Job?) : JobSupport(true), CompletableJob { * } * ``` */ - override val handlesException: Boolean = handlesException() + override val handlesException: Boolean = handlesException(parent) + + /* + * Only after that we init parent job (which might freeze this object if parent is frozen). + */ + init { initParentJob(parent) } + + override val onCancelComplete get() = true + override fun complete() = makeCompleting(Unit) override fun completeExceptionally(exception: Throwable): Boolean = makeCompleting(CompletedExceptionally(exception)) @JsName("handlesExceptionF") - private fun handlesException(): Boolean { - var parentJob = (parentHandle as? ChildHandleNode)?.job ?: return false - while (true) { - if (parentJob.handlesException) return true - parentJob = (parentJob.parentHandle as? ChildHandleNode)?.job ?: return false - } + private tailrec fun handlesException(parent: Job?): Boolean { + val parentJob = (parent as? JobSupport) ?: return false + if (parentJob.handlesException) return true + return handlesException((parentJob.parentHandle as? ChildHandleNode)?.job) } } @@ -1471,7 +1498,7 @@ internal class ChildContinuation( @JvmField val child: CancellableContinuationImpl<*> ) : JobCancellingNode() { override fun invoke(cause: Throwable?) { - child.parentCancelled(child.getContinuationCancellationCause(job)) + child.parentCancelled(job) } } diff --git a/kotlinx-coroutines-core/common/src/Supervisor.kt b/kotlinx-coroutines-core/common/src/Supervisor.kt index 8411c5c65a..b8f8a03576 100644 --- a/kotlinx-coroutines-core/common/src/Supervisor.kt +++ b/kotlinx-coroutines-core/common/src/Supervisor.kt @@ -69,6 +69,6 @@ private class SupervisorJobImpl(parent: Job?) : JobImpl(parent) { private class SupervisorCoroutine( context: CoroutineContext, uCont: Continuation -) : ScopeCoroutine(context, uCont) { +) : ScopeCoroutine(context, uCont, true) { override fun childCancelled(cause: Throwable): Boolean = false } diff --git a/kotlinx-coroutines-core/common/src/Timeout.kt b/kotlinx-coroutines-core/common/src/Timeout.kt index 46ab4ae8c8..99e6d99479 100644 --- a/kotlinx-coroutines-core/common/src/Timeout.kt +++ b/kotlinx-coroutines-core/common/src/Timeout.kt @@ -104,7 +104,7 @@ public suspend fun withTimeoutOrNull(timeMillis: Long, block: suspend Corout } } catch (e: TimeoutCancellationException) { // Return null if it's our exception, otherwise propagate it upstream (e.g. in case of nested withTimeouts) - if (e.coroutine === coroutine) { + if (e.coroutine.unweakRef() === coroutine) { return null } throw e @@ -149,7 +149,12 @@ private fun setupTimeout( private class TimeoutCoroutine( @JvmField val time: Long, uCont: Continuation // unintercepted continuation -) : ScopeCoroutine(uCont.context, uCont), Runnable { +) : ScopeCoroutine(uCont.context, uCont, false), Runnable { + init { + // Kludge for native + if (!isReuseSupportedInPlatform()) initParentForNativeUndispatchedCoroutine() + } + override fun run() { cancelCoroutine(TimeoutCancellationException(time, this)) } @@ -163,7 +168,7 @@ private class TimeoutCoroutine( */ public class TimeoutCancellationException internal constructor( message: String, - @JvmField internal val coroutine: Job? + @JvmField internal val coroutine: Any? ) : CancellationException(message), CopyableThrowable { /** * Creates a timeout exception with the given message. @@ -181,4 +186,4 @@ public class TimeoutCancellationException internal constructor( internal fun TimeoutCancellationException( time: Long, coroutine: Job -) : TimeoutCancellationException = TimeoutCancellationException("Timed out waiting for $time ms", coroutine) +) : TimeoutCancellationException = TimeoutCancellationException("Timed out waiting for $time ms", coroutine.weakRef()) diff --git a/kotlinx-coroutines-core/common/src/channels/AbstractChannel.kt b/kotlinx-coroutines-core/common/src/channels/AbstractChannel.kt index b92ced6ab7..4c3d885e62 100644 --- a/kotlinx-coroutines-core/common/src/channels/AbstractChannel.kt +++ b/kotlinx-coroutines-core/common/src/channels/AbstractChannel.kt @@ -80,7 +80,7 @@ internal abstract class AbstractSendChannel( * Returns non-null closed token if it is last in the queue. * @suppress **This is unstable API and it is subject to change.** */ - protected val closedForSend: Closed<*>? get() = (queue.prevNode as? Closed<*>)?.also { helpClose(it) } + protected val closedForSend: Closed<*>? get() = (queueTail() as? Closed<*>)?.also { helpClose(it) } /** * Returns non-null closed token if it is first in the queue. @@ -117,9 +117,9 @@ internal abstract class AbstractSendChannel( queue: LockFreeLinkedListHead, element: E ) : AddLastDesc>(queue, SendBuffered(element)) { - override fun failure(affected: LockFreeLinkedListNode): Any? = when (affected) { + override fun failure(affected: LockFreeLinkedListNode?): Any? = when (affected) { is Closed<*> -> affected - is ReceiveOrClosed<*> -> OFFER_FAILED + is ReceiveOrClosed<*>? -> OFFER_FAILED // must fail on null for unlinked nodes on K/N else -> null } } @@ -267,7 +267,8 @@ internal abstract class AbstractSendChannel( * "if (!close()) next send will throw" */ val closeAdded = queue.addLastIfPrev(closed) { it !is Closed<*> } - val actuallyClosed = if (closeAdded) closed else queue.prevNode as Closed<*> + val actuallyClosed = if (closeAdded) closed else queueTail() as Closed<*> + disposeLockFreeLinkedList { closed.takeUnless { closeAdded } } helpClose(actuallyClosed) if (closeAdded) invokeOnCloseHandler(cause) return closeAdded // true if we have closed @@ -342,6 +343,12 @@ internal abstract class AbstractSendChannel( closedList.forEachReversed { it.resumeReceiveClosed(closed) } // and do other post-processing onClosedIdempotent(closed) + // dispose on Kotlin/Native if closed is the only element in the queue now + disposeQueue { closed } + } + + internal inline fun disposeQueue(closed: () -> Closed<*>?) { + disposeLockFreeLinkedList { queue.takeIf { it.nextNode === closed() } } } /** @@ -371,9 +378,9 @@ internal abstract class AbstractSendChannel( @JvmField val element: E, queue: LockFreeLinkedListHead ) : RemoveFirstDesc>(queue) { - override fun failure(affected: LockFreeLinkedListNode): Any? = when (affected) { + override fun failure(affected: LockFreeLinkedListNode?): Any? = when (affected) { is Closed<*> -> affected - !is ReceiveOrClosed<*> -> OFFER_FAILED + !is ReceiveOrClosed<*> -> OFFER_FAILED // must fail on null for unlinked nodes on K/N else -> null } @@ -442,7 +449,7 @@ internal abstract class AbstractSendChannel( is Send -> "SendQueued" else -> "UNEXPECTED:$head" // should not happen } - val tail = queue.prevNode + val tail = queueTail() if (tail !== head) { result += ",queueSize=${countQueueSize()}" if (tail is Closed<*>) result += ",closedForSend=$tail" @@ -450,6 +457,13 @@ internal abstract class AbstractSendChannel( return result } + private fun queueTail(): LockFreeLinkedListNode { + // Backwards links can be already unlinked on Kotlin/Native when it was closed and only + // a closed node remains in it. queue.prevNode returns queue in this case + val tail = queue.prevNode + return if (tail === queue) queue.nextNode else tail + } + private fun countQueueSize(): Int { var size = 0 queue.forEach { size++ } @@ -464,13 +478,15 @@ internal abstract class AbstractSendChannel( override val pollResult: E, // E | Closed - the result pollInternal returns when it rendezvous with this node @JvmField val channel: AbstractSendChannel, @JvmField val select: SelectInstance, - @JvmField val block: suspend (SendChannel) -> R + block: suspend (SendChannel) -> R ) : Send(), DisposableHandle { + @JvmField val block: suspend (SendChannel) -> R = block.asShareable() + override fun tryResumeSend(otherOp: PrepareOp?): Symbol? = select.trySelectOther(otherOp) as Symbol? // must return symbol override fun completeResumeSend() { - block.startCoroutineCancellable(receiver = channel, completion = select.completion) + startCoroutine(CoroutineStart.DEFAULT, channel, select.completion, block = block) } override fun dispose() { // invoked on select completion @@ -540,6 +556,7 @@ internal abstract class AbstractChannel( protected open fun pollInternal(): Any? { while (true) { val send = takeFirstSendOrPeekClosed() ?: return POLL_FAILED + disposeQueue { send as? Closed<*> } val token = send.tryResumeSend(null) if (token != null) { assert { token === RESUME_TOKEN } @@ -660,6 +677,8 @@ internal abstract class AbstractChannel( internal fun cancelInternal(cause: Throwable?): Boolean = close(cause).also { onCancelIdempotent(it) + // dispose on Kotlin/Native if closed is the only element in the queue now + disposeQueue { closedForSend } } /** @@ -675,9 +694,8 @@ internal abstract class AbstractChannel( var list = InlineList() while (true) { val previous = closed.prevNode - if (previous is LockFreeLinkedListHead) { - break - } + // It could be already unlinked on Kotlin/Native and close.prevNode === closed + if (previous is LockFreeLinkedListHead || previous === closed) break assert { previous is Send } if (!previous.remove()) { previous.helpRemove() // make sure remove is complete before continuing @@ -709,9 +727,9 @@ internal abstract class AbstractChannel( * @suppress **This is unstable API and it is subject to change.** */ protected class TryPollDesc(queue: LockFreeLinkedListHead) : RemoveFirstDesc(queue) { - override fun failure(affected: LockFreeLinkedListNode): Any? = when (affected) { + override fun failure(affected: LockFreeLinkedListNode?): Any? = when (affected) { is Closed<*> -> affected - !is Send -> POLL_FAILED + !is Send -> POLL_FAILED // must fail on null for unlinked nodes on K/N else -> null } @@ -830,7 +848,10 @@ internal abstract class AbstractChannel( } private class Itr(@JvmField val channel: AbstractChannel) : ChannelIterator { - var result: Any? = POLL_FAILED // E | POLL_FAILED | Closed + private val _result = atomic(POLL_FAILED) // E | POLL_FAILED | Closed + var result: Any? + get() = _result.value + set(value) { _result.value = value } override suspend fun hasNext(): Boolean { // check for repeated hasNext @@ -889,30 +910,35 @@ internal abstract class AbstractChannel( } private open class ReceiveElement( - @JvmField val cont: CancellableContinuation, + cont: CancellableContinuation, @JvmField val receiveMode: Int ) : Receive() { + private val _cont = atomic?>(cont) + protected val cont get() = _cont.value!! + fun resumeValue(value: E): Any? = when (receiveMode) { RECEIVE_RESULT -> ChannelResult.success(value) else -> value } override fun tryResumeReceive(value: E, otherOp: PrepareOp?): Symbol? { - val token = cont.tryResume(resumeValue(value), otherOp?.desc, resumeOnCancellationFun(value)) ?: return null + val token = _cont.value?.tryResume(resumeValue(value), otherOp?.desc, resumeOnCancellationFun(value)) ?: return null assert { token === RESUME_TOKEN } // the only other possible result // We can call finishPrepare only after successful tryResume, so that only good affected node is saved otherOp?.finishPrepare() return RESUME_TOKEN } - override fun completeResumeReceive(value: E) = cont.completeResume(RESUME_TOKEN) + override fun completeResumeReceive(value: E) { _cont.getAndSet(null)!!.completeResume(RESUME_TOKEN) } override fun resumeReceiveClosed(closed: Closed<*>) { + val cont = _cont.getAndSet(null)!! when { receiveMode == RECEIVE_RESULT -> cont.resume(closed.toResult()) else -> cont.resumeWithException(closed.receiveException) } } + override fun toString(): String = "ReceiveElement@$hexAddress[receiveMode=$receiveMode]" } @@ -921,16 +947,20 @@ internal abstract class AbstractChannel( receiveMode: Int, @JvmField val onUndeliveredElement: OnUndeliveredElement ) : ReceiveElement(cont, receiveMode) { + private val context = cont.context override fun resumeOnCancellationFun(value: E): ((Throwable) -> Unit)? = - onUndeliveredElement.bindCancellationFun(value, cont.context) + onUndeliveredElement.bindCancellationFun(value, context) } private open class ReceiveHasNext( @JvmField val iterator: Itr, - @JvmField val cont: CancellableContinuation + cont: CancellableContinuation ) : Receive() { + private val _cont = atomic?>(cont) + private val context = cont.context + override fun tryResumeReceive(value: E, otherOp: PrepareOp?): Symbol? { - val token = cont.tryResume(true, otherOp?.desc, resumeOnCancellationFun(value)) + val token = _cont.value?.tryResume(true, otherOp?.desc, resumeOnCancellationFun(value)) ?: return null assert { token === RESUME_TOKEN } // the only other possible result // We can call finishPrepare only after successful tryResume, so that only good affected node is saved @@ -944,10 +974,11 @@ internal abstract class AbstractChannel( but completeResumeReceive is called once so we set iterator result here. */ iterator.result = value - cont.completeResume(RESUME_TOKEN) + _cont.getAndSet(null)!!.completeResume(RESUME_TOKEN) } override fun resumeReceiveClosed(closed: Closed<*>) { + val cont = _cont.getAndSet(null)!! val token = if (closed.closeCause == null) { cont.tryResume(false) } else { @@ -960,7 +991,7 @@ internal abstract class AbstractChannel( } override fun resumeOnCancellationFun(value: E): ((Throwable) -> Unit)? = - iterator.channel.onUndeliveredElement?.bindCancellationFun(value, cont.context) + iterator.channel.onUndeliveredElement?.bindCancellationFun(value, context) override fun toString(): String = "ReceiveHasNext@$hexAddress" } @@ -968,26 +999,24 @@ internal abstract class AbstractChannel( private class ReceiveSelect( @JvmField val channel: AbstractChannel, @JvmField val select: SelectInstance, - @JvmField val block: suspend (Any?) -> R, + block: suspend (Any?) -> R, @JvmField val receiveMode: Int ) : Receive(), DisposableHandle { + @JvmField val block: suspend (Any?) -> R = block.asShareable() // captured variables in this block need screening + override fun tryResumeReceive(value: E, otherOp: PrepareOp?): Symbol? = select.trySelectOther(otherOp) as Symbol? @Suppress("UNCHECKED_CAST") override fun completeResumeReceive(value: E) { - block.startCoroutineCancellable( - if (receiveMode == RECEIVE_RESULT) ChannelResult.success(value) else value, - select.completion, - resumeOnCancellationFun(value) - ) + startCoroutine(CoroutineStart.DEFAULT, if (receiveMode == RECEIVE_RESULT) ChannelResult.success(value) else value, select.completion, resumeOnCancellationFun(value), block) } override fun resumeReceiveClosed(closed: Closed<*>) { if (!select.trySelect()) return when (receiveMode) { RECEIVE_THROWS_ON_CLOSE -> select.resumeSelectWithException(closed.receiveException) - RECEIVE_RESULT -> block.startCoroutineCancellable(ChannelResult.closed(closed.closeCause), select.completion) + RECEIVE_RESULT -> startCoroutine(CoroutineStart.DEFAULT, ChannelResult.closed(closed.closeCause), select.completion, block = block) } } @@ -1064,28 +1093,33 @@ internal interface ReceiveOrClosed { /** * Represents sender for a specific element. */ -internal open class SendElement( - override val pollResult: E, - @JvmField val cont: CancellableContinuation +@Suppress("UNCHECKED_CAST") +internal open class SendElement( + override val pollResult: Any?, + cont: CancellableContinuation ) : Send() { + private val _cont = atomic?>(cont) + protected val cont get() = _cont.value!! + override fun tryResumeSend(otherOp: PrepareOp?): Symbol? { - val token = cont.tryResume(Unit, otherOp?.desc) ?: return null + val token = _cont.value?.tryResume(Unit, otherOp?.desc) ?: return null assert { token === RESUME_TOKEN } // the only other possible result // We can call finishPrepare only after successful tryResume, so that only good affected node is saved otherOp?.finishPrepare() // finish preparations return RESUME_TOKEN } - - override fun completeResumeSend() = cont.completeResume(RESUME_TOKEN) - override fun resumeSendClosed(closed: Closed<*>) = cont.resumeWithException(closed.sendException) - override fun toString(): String = "$classSimpleName@$hexAddress($pollResult)" + override fun completeResumeSend() { _cont.getAndSet(null)!!.completeResume(RESUME_TOKEN) } + override fun resumeSendClosed(closed: Closed<*>) { _cont.getAndSet(null)!!.resumeWithException(closed.sendException) } + override fun toString(): String = "SendElement@$hexAddress($pollResult)" } internal class SendElementWithUndeliveredHandler( pollResult: E, cont: CancellableContinuation, @JvmField val onUndeliveredElement: OnUndeliveredElement -) : SendElement(pollResult, cont) { +) : SendElement(pollResult, cont) { + private val context = cont.context + override fun remove(): Boolean { if (!super.remove()) return false // if the node was successfully removed (meaning it was added but was not received) then we have undelivered element @@ -1094,10 +1128,9 @@ internal class SendElementWithUndeliveredHandler( } override fun undeliveredElement() { - onUndeliveredElement.callUndeliveredElement(pollResult, cont.context) + onUndeliveredElement.callUndeliveredElement(pollResult as E, context) } } - /** * Represents closed channel. */ diff --git a/kotlinx-coroutines-core/common/src/channels/ArrayBroadcastChannel.kt b/kotlinx-coroutines-core/common/src/channels/ArrayBroadcastChannel.kt index 0a96f75380..9cb835cba3 100644 --- a/kotlinx-coroutines-core/common/src/channels/ArrayBroadcastChannel.kt +++ b/kotlinx-coroutines-core/common/src/channels/ArrayBroadcastChannel.kt @@ -5,9 +5,11 @@ package kotlinx.coroutines.channels import kotlinx.atomicfu.* +import kotlinx.atomicfu.locks.* import kotlinx.coroutines.* import kotlinx.coroutines.internal.* import kotlinx.coroutines.selects.* +import kotlinx.atomicfu.locks.ReentrantLock /** * Broadcast channel with array buffer of a fixed [capacity]. @@ -44,8 +46,7 @@ internal class ArrayBroadcastChannel( * - Read "tail" (volatile), then read element from buffer * So read/writes to buffer need not be volatile */ - private val bufferLock = ReentrantLock() - private val buffer = arrayOfNulls(capacity) + private val state = ArrayBufferState(capacity) // head & tail are Long (64 bits) and we assume that they never wrap around // head, tail, and size are guarded by bufferLock @@ -97,13 +98,13 @@ internal class ArrayBroadcastChannel( // result is `OFFER_SUCCESS | OFFER_FAILED | Closed` override fun offerInternal(element: E): Any { - bufferLock.withLock { + state.withLock { // check if closed for send (under lock, so size cannot change) closedForSend?.let { return it } val size = this.size if (size >= capacity) return OFFER_FAILED val tail = this.tail - buffer[(tail % capacity).toInt()] = element + state.setBufferAt((tail % capacity).toInt(), element) this.size = size + 1 this.tail = tail + 1 } @@ -114,7 +115,7 @@ internal class ArrayBroadcastChannel( // result is `ALREADY_SELECTED | OFFER_SUCCESS | OFFER_FAILED | Closed` override fun offerSelectInternal(element: E, select: SelectInstance<*>): Any { - bufferLock.withLock { + state.withLock { // check if closed for send (under lock, so size cannot change) closedForSend?.let { return it } val size = this.size @@ -124,7 +125,7 @@ internal class ArrayBroadcastChannel( return ALREADY_SELECTED } val tail = this.tail - buffer[(tail % capacity).toInt()] = element + state.setBufferAt((tail % capacity).toInt(), element) this.size = size + 1 this.tail = tail + 1 } @@ -149,7 +150,7 @@ internal class ArrayBroadcastChannel( private tailrec fun updateHead(addSub: Subscriber? = null, removeSub: Subscriber? = null) { // update head in a tail rec loop var send: Send? = null - bufferLock.withLock { + state.withLock { if (addSub != null) { addSub.subHead = tail // start from last element val wasEmpty = subscribers.isEmpty() @@ -168,7 +169,7 @@ internal class ArrayBroadcastChannel( var size = this.size // clean up removed (on not need if we don't have any subscribers anymore) while (head < targetHead) { - buffer[(head % capacity).toInt()] = null + state.setBufferAt((head % capacity).toInt(), null) val wasFull = size >= capacity // update the size before checking queue (no more senders can queue up) this.head = ++head @@ -181,7 +182,7 @@ internal class ArrayBroadcastChannel( if (token != null) { assert { token === RESUME_TOKEN } // put sent element to the buffer - buffer[(tail % capacity).toInt()] = (send as Send).pollResult + state.setBufferAt((tail % capacity).toInt(), (send as Send).pollResult) this.size = size + 1 this.tail = tail + 1 return@withLock // go out of lock to wakeup this sender @@ -208,13 +209,10 @@ internal class ArrayBroadcastChannel( return minHead } - @Suppress("UNCHECKED_CAST") - private fun elementAt(index: Long): E = buffer[(index % capacity).toInt()] as E - private class Subscriber( private val broadcastChannel: ArrayBroadcastChannel ) : AbstractChannel(null), ReceiveChannel { - private val subLock = ReentrantLock() + private val subLock = reentrantLock() private val _subHead = atomic(0L) var subHead: Long // guarded by subLock @@ -368,7 +366,8 @@ internal class ArrayBroadcastChannel( } // Get tentative result. This result may be wrong (completely invalid value, including null), // because this subscription might get closed, moving channel's head past this subscription's head. - val result = broadcastChannel.elementAt(subHead) + @Suppress("UNCHECKED_CAST") + val result = broadcastChannel.state.getBufferAt((subHead % broadcastChannel.capacity).toInt()) as E // now check if this subscription was closed val closedSub = this.closedForReceive if (closedSub != null) return closedSub @@ -380,5 +379,7 @@ internal class ArrayBroadcastChannel( // ------ debug ------ override val bufferDebugString: String - get() = "(buffer:capacity=${buffer.size},size=$size)" + get() = state.withLock { + "(buffer:capacity=${state.bufferSize},size=$size)" + } } diff --git a/kotlinx-coroutines-core/common/src/channels/ArrayBufferState.common.kt b/kotlinx-coroutines-core/common/src/channels/ArrayBufferState.common.kt new file mode 100644 index 0000000000..bd522e6e18 --- /dev/null +++ b/kotlinx-coroutines-core/common/src/channels/ArrayBufferState.common.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +internal expect open class ArrayBufferState(initialBufferSize: Int) { + val bufferSize: Int + + fun getBufferAt(index: Int): Any? + fun setBufferAt(index: Int, value: Any?) + + inline fun withLock(block: () -> T): T +} diff --git a/kotlinx-coroutines-core/common/src/channels/ArrayChannel.kt b/kotlinx-coroutines-core/common/src/channels/ArrayChannel.kt index 7e6c0e68c5..d01ef039ef 100644 --- a/kotlinx-coroutines-core/common/src/channels/ArrayChannel.kt +++ b/kotlinx-coroutines-core/common/src/channels/ArrayChannel.kt @@ -4,7 +4,6 @@ package kotlinx.coroutines.channels -import kotlinx.atomicfu.* import kotlinx.coroutines.* import kotlinx.coroutines.internal.* import kotlinx.coroutines.selects.* @@ -33,30 +32,24 @@ internal open class ArrayChannel( require(capacity >= 1) { "ArrayChannel capacity must be at least 1, but $capacity was specified" } } - private val lock = ReentrantLock() - /* - * Guarded by lock. - * Allocate minimum of capacity and 16 to avoid excess memory pressure for large channels when it's not necessary. + * Allocate minimum of capacity and 8 to avoid excess memory pressure for large channels when it's not necessary. */ - private var buffer: Array = arrayOfNulls(min(capacity, 8)).apply { fill(EMPTY) } - - private var head: Int = 0 - private val size = atomic(0) // Invariant: size <= capacity + private val state = ArrayChannelState(min(capacity, 8)) protected final override val isBufferAlwaysEmpty: Boolean get() = false - protected final override val isBufferEmpty: Boolean get() = size.value == 0 + protected final override val isBufferEmpty: Boolean get() = state.size == 0 protected final override val isBufferAlwaysFull: Boolean get() = false - protected final override val isBufferFull: Boolean get() = size.value == capacity && onBufferOverflow == BufferOverflow.SUSPEND + protected final override val isBufferFull: Boolean get() = state.size == capacity - override val isEmpty: Boolean get() = lock.withLock { isEmptyImpl } - override val isClosedForReceive: Boolean get() = lock.withLock { super.isClosedForReceive } + override val isEmpty: Boolean get() = state.withLock { isEmptyImpl } + override val isClosedForReceive: Boolean get() = state.withLock { super.isClosedForReceive } // result is `OFFER_SUCCESS | OFFER_FAILED | Closed` protected override fun offerInternal(element: E): Any { var receive: ReceiveOrClosed? = null - lock.withLock { - val size = this.size.value + state.withLock { + val size = state.size closedForSend?.let { return it } // update size before checking queue (!!!) updateBufferSize(size)?.let { return it } @@ -65,13 +58,13 @@ internal open class ArrayChannel( loop@ while (true) { receive = takeFirstReceiveOrPeekClosed() ?: break@loop // break when no receivers queued if (receive is Closed) { - this.size.value = size // restore size + state.size = size // restore size return receive!! } val token = receive!!.tryResumeReceive(element, null) if (token != null) { assert { token === RESUME_TOKEN } - this.size.value = size // restore size + state.size = size // restore size return@withLock } } @@ -87,8 +80,8 @@ internal open class ArrayChannel( // result is `ALREADY_SELECTED | OFFER_SUCCESS | OFFER_FAILED | Closed` protected override fun offerSelectInternal(element: E, select: SelectInstance<*>): Any { var receive: ReceiveOrClosed? = null - lock.withLock { - val size = this.size.value + state.withLock { + val size = state.size closedForSend?.let { return it } // update size before checking queue (!!!) updateBufferSize(size)?.let { return it } @@ -99,14 +92,14 @@ internal open class ArrayChannel( val failure = select.performAtomicTrySelect(offerOp) when { failure == null -> { // offered successfully - this.size.value = size // restore size + state.size = size // restore size receive = offerOp.result return@withLock } failure === OFFER_FAILED -> break@loop // cannot offer -> Ok to queue to buffer failure === RETRY_ATOMIC -> {} // retry failure === ALREADY_SELECTED || failure is Closed<*> -> { - this.size.value = size // restore size + state.size = size // restore size return failure } else -> error("performAtomicTrySelect(describeTryOffer) returned $failure") @@ -115,7 +108,7 @@ internal open class ArrayChannel( } // let's try to select sending this element to buffer if (!select.trySelect()) { // :todo: move trySelect completion outside of lock - this.size.value = size // restore size + state.size = size // restore size return ALREADY_SELECTED } enqueueElement(size, element) @@ -126,7 +119,7 @@ internal open class ArrayChannel( return receive!!.offerResult } - override fun enqueueSend(send: Send): Any? = lock.withLock { + override fun enqueueSend(send: Send): Any? = state.withLock { super.enqueueSend(send) } @@ -134,7 +127,7 @@ internal open class ArrayChannel( // Result is `OFFER_SUCCESS | OFFER_FAILED | null` private fun updateBufferSize(currentSize: Int): Symbol? { if (currentSize < capacity) { - size.value = currentSize + 1 // tentatively put it into the buffer + state.size = currentSize + 1 // tentatively put it into the buffer return null // proceed } // buffer is full @@ -148,28 +141,15 @@ internal open class ArrayChannel( // Guarded by lock private fun enqueueElement(currentSize: Int, element: E) { if (currentSize < capacity) { - ensureCapacity(currentSize) - buffer[(head + currentSize) % buffer.size] = element // actually queue element + state.ensureCapacity(currentSize, capacity) + state.setBufferAt((state.head + currentSize) % state.bufferSize, element) // actually queue element } else { // buffer is full assert { onBufferOverflow == BufferOverflow.DROP_OLDEST } // the only way we can get here - buffer[head % buffer.size] = null // drop oldest element - buffer[(head + currentSize) % buffer.size] = element // actually queue element - head = (head + 1) % buffer.size - } - } - - // Guarded by lock - private fun ensureCapacity(currentSize: Int) { - if (currentSize >= buffer.size) { - val newSize = min(buffer.size * 2, capacity) - val newBuffer = arrayOfNulls(newSize) - for (i in 0 until currentSize) { - newBuffer[i] = buffer[(head + i) % buffer.size] - } - newBuffer.fill(EMPTY, currentSize, newSize) - buffer = newBuffer - head = 0 + state.setBufferAt(state.head % state.bufferSize, null) // drop oldest element + state.setBufferAt((state.head + currentSize) % state.bufferSize, element) // actually queue element + // actually queue element + state.head = (state.head + 1) % state.bufferSize } } @@ -178,18 +158,19 @@ internal open class ArrayChannel( var send: Send? = null var resumed = false var result: Any? = null - lock.withLock { - val size = this.size.value + state.withLock { + val size = state.size if (size == 0) return closedForSend ?: POLL_FAILED // when nothing can be read from buffer // size > 0: not empty -- retrieve element - result = buffer[head] - buffer[head] = null - this.size.value = size - 1 // update size before checking queue (!!!) + result = state.getBufferAt(state.head) + state.setBufferAt(state.head, null) + state.size = size - 1 // update size before checking queue (!!!) // check for senders that were waiting on full queue var replacement: Any? = POLL_FAILED if (size == capacity) { loop@ while (true) { send = takeFirstSendOrPeekClosed() ?: break + disposeQueue { send as? Closed<*> } val token = send!!.tryResumeSend(null) if (token != null) { assert { token === RESUME_TOKEN } @@ -202,10 +183,10 @@ internal open class ArrayChannel( } } if (replacement !== POLL_FAILED && replacement !is Closed<*>) { - this.size.value = size // restore size - buffer[(head + size) % buffer.size] = replacement + state.size = size // restore size + state.setBufferAt((state.head + size) % state.bufferSize, replacement) } - head = (head + 1) % buffer.size + state.head = (state.head + 1) % state.bufferSize } // complete send the we're taken replacement from if (resumed) @@ -218,13 +199,13 @@ internal open class ArrayChannel( var send: Send? = null var success = false var result: Any? = null - lock.withLock { - val size = this.size.value + state.withLock { + val size = state.size if (size == 0) return closedForSend ?: POLL_FAILED // size > 0: not empty -- retrieve element - result = buffer[head] - buffer[head] = null - this.size.value = size - 1 // update size before checking queue (!!!) + result = state.getBufferAt(state.head) + state.setBufferAt(state.head, null) + state.size = size - 1 // update size before checking queue (!!!) // check for senders that were waiting on full queue var replacement: Any? = POLL_FAILED if (size == capacity) { @@ -241,8 +222,8 @@ internal open class ArrayChannel( failure === POLL_FAILED -> break@loop // cannot poll -> Ok to take from buffer failure === RETRY_ATOMIC -> {} // retry failure === ALREADY_SELECTED -> { - this.size.value = size // restore size - buffer[head] = result // restore head + state.size = size // restore size + state.setBufferAt(state.head, result) // restore head return failure } failure is Closed<*> -> { @@ -256,17 +237,17 @@ internal open class ArrayChannel( } } if (replacement !== POLL_FAILED && replacement !is Closed<*>) { - this.size.value = size // restore size - buffer[(head + size) % buffer.size] = replacement + state.size = size // restore size + state.setBufferAt((state.head + size) % state.bufferSize, replacement) } else { // failed to poll or is already closed --> let's try to select receiving this element from buffer if (!select.trySelect()) { // :todo: move trySelect completion outside of lock - this.size.value = size // restore size - buffer[head] = result // restore head + state.size = size // restore size + state.setBufferAt(state.head, result) // restore head return ALREADY_SELECTED } } - head = (head + 1) % buffer.size + state.head = (state.head + 1) % state.bufferSize } // complete send the we're taken replacement from if (success) @@ -274,7 +255,7 @@ internal open class ArrayChannel( return result } - override fun enqueueReceiveInternal(receive: Receive): Boolean = lock.withLock { + override fun enqueueReceiveInternal(receive: Receive): Boolean = state.withLock { super.enqueueReceiveInternal(receive) } @@ -283,17 +264,17 @@ internal open class ArrayChannel( // clear buffer first, but do not wait for it in helpers val onUndeliveredElement = onUndeliveredElement var undeliveredElementException: UndeliveredElementException? = null // first cancel exception, others suppressed - lock.withLock { - repeat(size.value) { - val value = buffer[head] + state.withLock { + repeat(state.size) { + val value = state.getBufferAt(state.head) if (onUndeliveredElement != null && value !== EMPTY) { @Suppress("UNCHECKED_CAST") undeliveredElementException = onUndeliveredElement.callUndeliveredElementCatchingException(value as E, undeliveredElementException) } - buffer[head] = EMPTY - head = (head + 1) % buffer.size + state.setBufferAt(state.head, null) + state.head = (state.head + 1) % state.bufferSize } - size.value = 0 + state.size = 0 } // then clean all queued senders super.onCancelIdempotent(wasClosed) @@ -303,5 +284,7 @@ internal open class ArrayChannel( // ------ debug ------ override val bufferDebugString: String - get() = "(buffer:capacity=$capacity,size=${size.value})" + get() = state.withLock { + "(buffer:capacity=$capacity,size=${state.size})" + } } diff --git a/kotlinx-coroutines-core/common/src/channels/ArrayChannelState.common.kt b/kotlinx-coroutines-core/common/src/channels/ArrayChannelState.common.kt new file mode 100644 index 0000000000..682d27641f --- /dev/null +++ b/kotlinx-coroutines-core/common/src/channels/ArrayChannelState.common.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +internal expect class ArrayChannelState(initialBufferSize: Int) : ArrayBufferState { + var head: Int + var size: Int // Invariant: size <= capacity + + fun ensureCapacity(currentSize: Int, capacity: Int) +} diff --git a/kotlinx-coroutines-core/common/src/channels/Broadcast.kt b/kotlinx-coroutines-core/common/src/channels/Broadcast.kt index b1c24b456d..7f96b77921 100644 --- a/kotlinx-coroutines-core/common/src/channels/Broadcast.kt +++ b/kotlinx-coroutines-core/common/src/channels/Broadcast.kt @@ -7,9 +7,11 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel.Factory.CONFLATED import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED -import kotlinx.coroutines.intrinsics.* +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.isReuseSupportedInPlatform import kotlin.coroutines.* import kotlin.coroutines.intrinsics.* +import kotlin.native.concurrent.* /** * Broadcasts all elements of the channel. @@ -136,7 +138,11 @@ private open class BroadcastCoroutine( ProducerScope, BroadcastChannel by _channel { init { - initParentJob(parentContext[Job]) + if (isReuseSupportedInPlatform()) initParentJob(parentContext[Job]) + } + + override fun onStart() { + if (!isReuseSupportedInPlatform()) initParentJob(parentContext[Job]) } override val isActive: Boolean get() = super.isActive @@ -182,7 +188,7 @@ private class LazyBroadcastCoroutine( channel: BroadcastChannel, block: suspend ProducerScope.() -> Unit ) : BroadcastCoroutine(parentContext, channel, active = false) { - private val continuation = block.createCoroutineUnintercepted(this, this) + private val saved = saveLazyCoroutine(this, this, block) override fun openSubscription(): ReceiveChannel { // open subscription _first_ @@ -193,6 +199,6 @@ private class LazyBroadcastCoroutine( } override fun onStart() { - continuation.startCoroutineCancellable(this) + startLazyCoroutine(saved, this, this) } } diff --git a/kotlinx-coroutines-core/common/src/channels/ConflatedChannel.kt b/kotlinx-coroutines-core/common/src/channels/ConflatedChannel.kt index 177e80cb49..e7ade78db2 100644 --- a/kotlinx-coroutines-core/common/src/channels/ConflatedChannel.kt +++ b/kotlinx-coroutines-core/common/src/channels/ConflatedChannel.kt @@ -7,6 +7,8 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* import kotlinx.coroutines.internal.* import kotlinx.coroutines.selects.* +import kotlin.jvm.* +import kotlin.native.concurrent.* /** * Channel that buffers at most one element and conflates all subsequent `send` and `trySend` invocations, @@ -19,23 +21,21 @@ import kotlinx.coroutines.selects.* */ internal open class ConflatedChannel(onUndeliveredElement: OnUndeliveredElement?) : AbstractChannel(onUndeliveredElement) { protected final override val isBufferAlwaysEmpty: Boolean get() = false - protected final override val isBufferEmpty: Boolean get() = lock.withLock { value === EMPTY } + protected final override val isBufferEmpty: Boolean get() = state.withLock { state.value === EMPTY } protected final override val isBufferAlwaysFull: Boolean get() = false protected final override val isBufferFull: Boolean get() = false - override val isEmpty: Boolean get() = lock.withLock { isEmptyImpl } + override val isEmpty: Boolean get() = state.withLock { isEmptyImpl } - private val lock = ReentrantLock() - - private var value: Any? = EMPTY + private val state = ConflatedChannelState() // result is `OFFER_SUCCESS | Closed` protected override fun offerInternal(element: E): Any { var receive: ReceiveOrClosed? = null - lock.withLock { + state.withLock { closedForSend?.let { return it } // if there is no element written in buffer - if (value === EMPTY) { + if (state.value === EMPTY) { // check for receivers that were waiting on the empty buffer loop@ while(true) { receive = takeFirstReceiveOrPeekClosed() ?: break@loop // break when no receivers queued @@ -60,9 +60,9 @@ internal open class ConflatedChannel(onUndeliveredElement: OnUndeliveredEleme // result is `ALREADY_SELECTED | OFFER_SUCCESS | Closed` protected override fun offerSelectInternal(element: E, select: SelectInstance<*>): Any { var receive: ReceiveOrClosed? = null - lock.withLock { + state.withLock { closedForSend?.let { return it } - if (value === EMPTY) { + if (state.value === EMPTY) { loop@ while(true) { val offerOp = describeTryOffer(element) val failure = select.performAtomicTrySelect(offerOp) @@ -93,10 +93,10 @@ internal open class ConflatedChannel(onUndeliveredElement: OnUndeliveredEleme // result is `E | POLL_FAILED | Closed` protected override fun pollInternal(): Any? { var result: Any? = null - lock.withLock { - if (value === EMPTY) return closedForSend ?: POLL_FAILED - result = value - value = EMPTY + state.withLock { + if (state.value === EMPTY) return closedForSend ?: POLL_FAILED + result = state.value + state.value = EMPTY } return result } @@ -104,19 +104,19 @@ internal open class ConflatedChannel(onUndeliveredElement: OnUndeliveredEleme // result is `E | POLL_FAILED | Closed` protected override fun pollSelectInternal(select: SelectInstance<*>): Any? { var result: Any? = null - lock.withLock { - if (value === EMPTY) return closedForSend ?: POLL_FAILED + state.withLock { + if (state.value === EMPTY) return closedForSend ?: POLL_FAILED if (!select.trySelect()) return ALREADY_SELECTED - result = value - value = EMPTY + result = state.value + state.value = EMPTY } return result } protected override fun onCancelIdempotent(wasClosed: Boolean) { var undeliveredElementException: UndeliveredElementException? = null // resource cancel exception - lock.withLock { + state.withLock { undeliveredElementException = updateValueLocked(EMPTY) } super.onCancelIdempotent(wasClosed) @@ -125,19 +125,19 @@ internal open class ConflatedChannel(onUndeliveredElement: OnUndeliveredEleme @Suppress("UNCHECKED_CAST") private fun updateValueLocked(element: Any?): UndeliveredElementException? { - val old = value + val old = state.value val undeliveredElementException = if (old === EMPTY) null else onUndeliveredElement?.callUndeliveredElementCatchingException(old as E) - value = element + state.value = element return undeliveredElementException } - override fun enqueueReceiveInternal(receive: Receive): Boolean = lock.withLock { + override fun enqueueReceiveInternal(receive: Receive): Boolean = state.withLock { super.enqueueReceiveInternal(receive) } // ------ debug ------ override val bufferDebugString: String - get() = lock.withLock { "(value=$value)" } + get() = state.withLock { "(value=${state.value})" } } diff --git a/kotlinx-coroutines-core/common/src/channels/ConflatedChannelState.common.kt b/kotlinx-coroutines-core/common/src/channels/ConflatedChannelState.common.kt new file mode 100644 index 0000000000..c9b7138790 --- /dev/null +++ b/kotlinx-coroutines-core/common/src/channels/ConflatedChannelState.common.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +internal expect class ConflatedChannelState() { + var value: Any? + inline fun withLock(block: () -> T): T +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/common/src/flow/SharedFlow.kt b/kotlinx-coroutines-core/common/src/flow/SharedFlow.kt index 0a291f258f..11e8a6281f 100644 --- a/kotlinx-coroutines-core/common/src/flow/SharedFlow.kt +++ b/kotlinx-coroutines-core/common/src/flow/SharedFlow.kt @@ -4,17 +4,19 @@ package kotlinx.coroutines.flow +import kotlinx.atomicfu.* import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.flow.internal.* import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.synchronized import kotlin.coroutines.* import kotlin.jvm.* import kotlin.native.concurrent.* /** * A _hot_ [Flow] that shares emitted values among all its collectors in a broadcast fashion, so that all collectors - * get all emitted values. A shared flow is called _hot_ because its active instance exists independently of the + * get all emitted values. A shared flow is called _hot_ because its active instance exists independently of the * presence of collectors. This is opposed to a regular [Flow], such as defined by the [`flow { ... }`][flow] function, * which is _cold_ and is started separately for each collector. * @@ -279,23 +281,20 @@ public fun MutableSharedFlow( // ------------------------------------ Implementation ------------------------------------ internal class SharedFlowSlot : AbstractSharedFlowSlot>() { - @JvmField - var index = -1L // current "to-be-emitted" index, -1 means the slot is free now - - @JvmField - var cont: Continuation? = null // collector waiting for new value + val index = atomic(-1L) // current "to-be-emitted" index, -1 means the slot is free now + val cont = atomic?>(null) // collector waiting for new value override fun allocateLocked(flow: SharedFlowImpl<*>): Boolean { - if (index >= 0) return false // not free - index = flow.updateNewCollectorIndexLocked() + if (index.value >= 0) return false // not free + index.value = flow.updateNewCollectorIndexLocked() return true } override fun freeLocked(flow: SharedFlowImpl<*>): Array?> { - assert { index >= 0 } - val oldIndex = index - index = -1L - cont = null // cleanup continuation reference + assert { index.value >= 0 } + val oldIndex = index.value + index.value = -1L + cont.value = null // cleanup continuation reference return flow.updateCollectorIndexLocked(oldIndex) } } @@ -335,27 +334,27 @@ internal open class SharedFlowImpl( */ // Stored state - private var buffer: Array? = null // allocated when needed, allocated size always power of two - private var replayIndex = 0L // minimal index from which new collector gets values - private var minCollectorIndex = 0L // minimal index of active collectors, equal to replayIndex if there are none - private var bufferSize = 0 // number of buffered values - private var queueSize = 0 // number of queued emitters + private val buffer = atomic?>(null) // allocated when needed, allocated size always power of two + private val replayIndex = atomic(0L) // minimal index from which new collector gets values + private val minCollectorIndex = atomic(0L) // minimal index of active collectors, equal to replayIndex if there are none + private val bufferSize = atomic(0) // number of buffered values + private val queueSize = atomic(0) // number of queued emitters // Computed state - private val head: Long get() = minOf(minCollectorIndex, replayIndex) - private val replaySize: Int get() = (head + bufferSize - replayIndex).toInt() - private val totalSize: Int get() = bufferSize + queueSize - private val bufferEndIndex: Long get() = head + bufferSize - private val queueEndIndex: Long get() = head + bufferSize + queueSize + private val head: Long get() = minOf(minCollectorIndex.value, replayIndex.value) + private val replaySize: Int get() = (head + bufferSize.value - replayIndex.value).toInt() + private val totalSize: Int get() = bufferSize.value + queueSize.value + private val bufferEndIndex: Long get() = head + bufferSize.value + private val queueEndIndex: Long get() = head + bufferSize.value + queueSize.value override val replayCache: List get() = synchronized(this) { val replaySize = this.replaySize if (replaySize == 0) return emptyList() val result = ArrayList(replaySize) - val buffer = buffer!! // must be allocated, because replaySize > 0 + val buffer = buffer.value!! // must be allocated, because replaySize > 0 @Suppress("UNCHECKED_CAST") - for (i in 0 until replaySize) result += buffer.getBufferAt(replayIndex + i) as T + for (i in 0 until replaySize) result += buffer.getBufferAt(replayIndex.value + i) as T result } @@ -364,7 +363,7 @@ internal open class SharedFlowImpl( */ @Suppress("UNCHECKED_CAST") protected val lastReplayedLocked: T - get() = buffer!!.getBufferAt(replayIndex + replaySize - 1) as T + get() = buffer.value!!.getBufferAt(replayIndex.value + replaySize - 1) as T @Suppress("UNCHECKED_CAST") override suspend fun collect(collector: FlowCollector): Nothing { @@ -412,7 +411,7 @@ internal open class SharedFlowImpl( if (nCollectors == 0) return tryEmitNoCollectorsLocked(value) // always returns true // With collectors we'll have to buffer // cannot emit now if buffer is full & blocked by slow collectors - if (bufferSize >= bufferCapacity && minCollectorIndex <= replayIndex) { + if (bufferSize.value >= bufferCapacity && minCollectorIndex.value <= replayIndex.value) { when (onBufferOverflow) { BufferOverflow.SUSPEND -> return false // will suspend BufferOverflow.DROP_LATEST -> return true // just drop incoming @@ -420,12 +419,12 @@ internal open class SharedFlowImpl( } } enqueueLocked(value) - bufferSize++ // value was added to buffer + bufferSize.incrementAndGet() // value was added to buffer // drop oldest from the buffer if it became more than bufferCapacity - if (bufferSize > bufferCapacity) dropOldestLocked() + if (bufferSize.value > bufferCapacity) dropOldestLocked() // keep replaySize not larger that needed if (replaySize > replay) { // increment replayIndex by one - updateBufferLocked(replayIndex + 1, minCollectorIndex, bufferEndIndex, queueEndIndex) + updateBufferLocked(replayIndex.value + 1, minCollectorIndex.value, bufferEndIndex, queueEndIndex) } return true } @@ -434,45 +433,46 @@ internal open class SharedFlowImpl( assert { nCollectors == 0 } if (replay == 0) return true // no need to replay, just forget it now enqueueLocked(value) // enqueue to replayCache - bufferSize++ // value was added to buffer + bufferSize.incrementAndGet() // value was added to buffer // drop oldest from the buffer if it became more than replay - if (bufferSize > replay) dropOldestLocked() - minCollectorIndex = head + bufferSize // a default value (max allowed) + if (bufferSize.value > replay) dropOldestLocked() + val newBufferSize = bufferSize.value // :KLUDGE: + minCollectorIndex.value = head + newBufferSize // a default value (max allowed) return true } private fun dropOldestLocked() { - buffer!!.setBufferAt(head, null) - bufferSize-- + buffer.value!!.setBufferAt(head, null) + bufferSize.decrementAndGet() val newHead = head + 1 - if (replayIndex < newHead) replayIndex = newHead - if (minCollectorIndex < newHead) correctCollectorIndexesOnDropOldest(newHead) + if (replayIndex.value < newHead) replayIndex.value = newHead + if (minCollectorIndex.value < newHead) correctCollectorIndexesOnDropOldest(newHead) assert { head == newHead } // since head = minOf(minCollectorIndex, replayIndex) it should have updated } private fun correctCollectorIndexesOnDropOldest(newHead: Long) { forEachSlotLocked { slot -> @Suppress("ConvertTwoComparisonsToRangeCheck") // Bug in JS backend - if (slot.index >= 0 && slot.index < newHead) { - slot.index = newHead // force move it up (this collector was too slow and missed the value at its index) + if (slot.index.value >= 0 && slot.index.value < newHead) { + slot.index.value = newHead // force move it up (this collector was too slow and missed the value at its index) } } - minCollectorIndex = newHead + minCollectorIndex.value = newHead } // enqueues item to buffer array, caller shall increment either bufferSize or queueSize private fun enqueueLocked(item: Any?) { val curSize = totalSize - val buffer = when (val curBuffer = buffer) { + val buffer = when (val curBuffer = buffer.value) { null -> growBuffer(null, 0, 2) else -> if (curSize >= curBuffer.size) growBuffer(curBuffer, curSize,curBuffer.size * 2) else curBuffer } buffer.setBufferAt(head + curSize, item) } - private fun growBuffer(curBuffer: Array?, curSize: Int, newSize: Int): Array { + private fun growBuffer(curBuffer: SharedFlowState?, curSize: Int, newSize: Int): SharedFlowState { check(newSize > 0) { "Buffer size overflow" } - val newBuffer = arrayOfNulls(newSize).also { buffer = it } + val newBuffer = SharedFlowState(newSize).also { buffer.value = it } if (curBuffer == null) return newBuffer val head = head for (i in 0 until curSize) { @@ -493,7 +493,7 @@ internal open class SharedFlowImpl( // add suspended emitter to the buffer Emitter(this, head + totalSize, value, cont).also { enqueueLocked(it) - queueSize++ // added to queue of waiting emitters + queueSize.incrementAndGet() // added to queue of waiting emitters // synchronous shared flow might rendezvous with waiting emitter if (bufferCapacity == 0) resumes = findSlotsToResumeLocked(resumes) } @@ -506,33 +506,33 @@ internal open class SharedFlowImpl( private fun cancelEmitter(emitter: Emitter) = synchronized(this) { if (emitter.index < head) return // already skipped past this index - val buffer = buffer!! + val buffer = buffer.value!! if (buffer.getBufferAt(emitter.index) !== emitter) return // already resumed buffer.setBufferAt(emitter.index, NO_VALUE) cleanupTailLocked() } internal fun updateNewCollectorIndexLocked(): Long { - val index = replayIndex - if (index < minCollectorIndex) minCollectorIndex = index + val index = replayIndex.value + if (index < minCollectorIndex.value) minCollectorIndex.value = index return index } // Is called when a collector disappears or changes index, returns a list of continuations to resume after lock internal fun updateCollectorIndexLocked(oldIndex: Long): Array?> { - assert { oldIndex >= minCollectorIndex } - if (oldIndex > minCollectorIndex) return EMPTY_RESUMES // nothing changes, it was not min + assert { oldIndex >= minCollectorIndex.value } + if (oldIndex > minCollectorIndex.value) return EMPTY_RESUMES // nothing changes, it was not min // start computing new minimal index of active collectors val head = head - var newMinCollectorIndex = head + bufferSize + var newMinCollectorIndex = head + bufferSize.value // take into account a special case of sync shared flow that can go past 1st queued emitter - if (bufferCapacity == 0 && queueSize > 0) newMinCollectorIndex++ + if (bufferCapacity == 0 && queueSize.value > 0) newMinCollectorIndex++ forEachSlotLocked { slot -> @Suppress("ConvertTwoComparisonsToRangeCheck") // Bug in JS backend - if (slot.index >= 0 && slot.index < newMinCollectorIndex) newMinCollectorIndex = slot.index + if (slot.index.value >= 0 && slot.index.value < newMinCollectorIndex) newMinCollectorIndex = slot.index.value } - assert { newMinCollectorIndex >= minCollectorIndex } // can only grow - if (newMinCollectorIndex <= minCollectorIndex) return EMPTY_RESUMES // nothing changes + assert { newMinCollectorIndex >= minCollectorIndex.value } // can only grow + if (newMinCollectorIndex <= minCollectorIndex.value) return EMPTY_RESUMES // nothing changes // Compute new buffer size if we drop items we no longer need and no emitter is resumed: // We must keep all the items from newMinIndex to the end of buffer var newBufferEndIndex = bufferEndIndex // var to grow when waiters are resumed @@ -541,17 +541,17 @@ internal open class SharedFlowImpl( // a) queueSize -> that's how many waiting emitters we have // b) bufferCapacity - newBufferSize0 -> that's how many we can afford to resume to add w/o exceeding bufferCapacity val newBufferSize0 = (newBufferEndIndex - newMinCollectorIndex).toInt() - minOf(queueSize, bufferCapacity - newBufferSize0) + minOf(queueSize.value, bufferCapacity - newBufferSize0) } else { // If we don't have collectors anymore we must resume all waiting emitters - queueSize // that's how many waiting emitters we have (at most) + queueSize.value // that's how many waiting emitters we have (at most) } var resumes: Array?> = EMPTY_RESUMES - val newQueueEndIndex = newBufferEndIndex + queueSize + val newQueueEndIndex = newBufferEndIndex + queueSize.value if (maxResumeCount > 0) { // collect emitters to resume if we have them resumes = arrayOfNulls(maxResumeCount) var resumeCount = 0 - val buffer = buffer!! + val buffer = buffer.value!! for (curEmitterIndex in newBufferEndIndex until newQueueEndIndex) { val emitter = buffer.getBufferAt(curEmitterIndex) if (emitter !== NO_VALUE) { @@ -573,9 +573,9 @@ internal open class SharedFlowImpl( // expression, which coerces values that are too big anyway. if (nCollectors == 0) newMinCollectorIndex = newBufferEndIndex // Compute new replay size -> limit to replay the number of items we need, take into account that it can only grow - var newReplayIndex = maxOf(replayIndex, newBufferEndIndex - minOf(replay, newBufferSize1)) + var newReplayIndex = maxOf(replayIndex.value, newBufferEndIndex - minOf(replay, newBufferSize1)) // adjustment for synchronous case with cancelled emitter (NO_VALUE) - if (bufferCapacity == 0 && newReplayIndex < newQueueEndIndex && buffer!!.getBufferAt(newReplayIndex) == NO_VALUE) { + if (bufferCapacity == 0 && newReplayIndex < newQueueEndIndex && buffer.value!!.getBufferAt(newReplayIndex) == NO_VALUE) { newBufferEndIndex++ newReplayIndex++ } @@ -598,25 +598,25 @@ internal open class SharedFlowImpl( val newHead = minOf(newMinCollectorIndex, newReplayIndex) assert { newHead >= head } // cleanup items we don't have to buffer anymore (because head is about to move) - for (index in head until newHead) buffer!!.setBufferAt(index, null) + for (index in head until newHead) buffer.value!!.setBufferAt(index, null) // update all state variables to newly computed values - replayIndex = newReplayIndex - minCollectorIndex = newMinCollectorIndex - bufferSize = (newBufferEndIndex - newHead).toInt() - queueSize = (newQueueEndIndex - newBufferEndIndex).toInt() + replayIndex.value = newReplayIndex + minCollectorIndex.value = newMinCollectorIndex + bufferSize.value = (newBufferEndIndex - newHead).toInt() + queueSize.value = (newQueueEndIndex - newBufferEndIndex).toInt() // check our key invariants (just in case) - assert { bufferSize >= 0 } - assert { queueSize >= 0 } - assert { replayIndex <= this.head + bufferSize } + assert { bufferSize.value >= 0 } + assert { queueSize.value >= 0 } + assert { replayIndex.value <= this.head + bufferSize.value } } // Removes all the NO_VALUE items from the end of the queue and reduces its size private fun cleanupTailLocked() { // If we have synchronous case, then keep one emitter queued - if (bufferCapacity == 0 && queueSize <= 1) return // return, don't clear it - val buffer = buffer!! - while (queueSize > 0 && buffer.getBufferAt(head + totalSize - 1) === NO_VALUE) { - queueSize-- + if (bufferCapacity == 0 && queueSize.value <= 1) return // return, don't clear it + val buffer = buffer.value!! + while (queueSize.value > 0 && buffer.getBufferAt(head + totalSize - 1) === NO_VALUE) { + queueSize.decrementAndGet() buffer.setBufferAt(head + totalSize, null) } } @@ -629,9 +629,9 @@ internal open class SharedFlowImpl( if (index < 0) { NO_VALUE } else { - val oldIndex = slot.index + val oldIndex = slot.index.value val newValue = getPeekedValueLockedAt(index) - slot.index = index + 1 // points to the next index after peeked one + slot.index.value = index + 1 // points to the next index after peeked one resumes = updateCollectorIndexLocked(oldIndex) newValue } @@ -643,17 +643,17 @@ internal open class SharedFlowImpl( // returns -1 if cannot peek value without suspension private fun tryPeekLocked(slot: SharedFlowSlot): Long { // return buffered value if possible - val index = slot.index + val index = slot.index.value if (index < bufferEndIndex) return index if (bufferCapacity > 0) return -1L // if there's a buffer, never try to rendezvous with emitters // Synchronous shared flow (bufferCapacity == 0) tries to rendezvous if (index > head) return -1L // ... but only with the first emitter (never look forward) - if (queueSize == 0) return -1L // nothing there to rendezvous with + if (queueSize.value == 0) return -1L // nothing there to rendezvous with return index // rendezvous with the first emitter } private fun getPeekedValueLockedAt(index: Long): Any? = - when (val item = buffer!!.getBufferAt(index)) { + when (val item = buffer.value!!.getBufferAt(index)) { is Emitter -> item.value else -> item } @@ -662,12 +662,12 @@ internal open class SharedFlowImpl( synchronized(this) lock@{ val index = tryPeekLocked(slot) // recheck under this lock if (index < 0) { - slot.cont = cont // Ok -- suspending + slot.cont.value = cont // Ok -- suspending } else { cont.resume(Unit) // has value, no need to suspend return@lock } - slot.cont = cont // suspend, waiting + slot.cont.value = cont // suspend, waiting } } @@ -675,23 +675,22 @@ internal open class SharedFlowImpl( var resumes: Array?> = resumesIn var resumeCount = resumesIn.size forEachSlotLocked loop@{ slot -> - val cont = slot.cont ?: return@loop // only waiting slots + val cont = slot.cont.value ?: return@loop // only waiting slots if (tryPeekLocked(slot) < 0) return@loop // only slots that can peek a value if (resumeCount >= resumes.size) resumes = resumes.copyOf(maxOf(2, 2 * resumes.size)) resumes[resumeCount++] = cont - slot.cont = null // not waiting anymore + slot.cont.value = null // not waiting anymore } return resumes } override fun createSlot() = SharedFlowSlot() - override fun createSlotArray(size: Int): Array = arrayOfNulls(size) override fun resetReplayCache() = synchronized(this) { // Update buffer state updateBufferLocked( newReplayIndex = bufferEndIndex, - newMinCollectorIndex = minCollectorIndex, + newMinCollectorIndex = minCollectorIndex.value, newBufferEndIndex = bufferEndIndex, newQueueEndIndex = queueEndIndex ) @@ -714,8 +713,6 @@ internal open class SharedFlowImpl( @JvmField internal val NO_VALUE = Symbol("NO_VALUE") -private fun Array.getBufferAt(index: Long) = get(index.toInt() and (size - 1)) -private fun Array.setBufferAt(index: Long, item: Any?) = set(index.toInt() and (size - 1), item) internal fun SharedFlow.fuseSharedFlow( context: CoroutineContext, diff --git a/kotlinx-coroutines-core/common/src/flow/StateFlow.kt b/kotlinx-coroutines-core/common/src/flow/StateFlow.kt index be6cbd6bbd..be7dca8886 100644 --- a/kotlinx-coroutines-core/common/src/flow/StateFlow.kt +++ b/kotlinx-coroutines-core/common/src/flow/StateFlow.kt @@ -5,10 +5,12 @@ package kotlinx.coroutines.flow import kotlinx.atomicfu.* +import kotlinx.atomicfu.locks.* import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.flow.internal.* import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.synchronized import kotlin.coroutines.* import kotlin.native.concurrent.* @@ -84,7 +86,7 @@ import kotlin.native.concurrent.* * * Use [SharedFlow] when you need a [StateFlow] with tweaks in its behavior such as extra buffering, replaying more * values, or omitting the initial value. - * + * * ### StateFlow vs ConflatedBroadcastChannel * * Conceptually, state flow is similar to [ConflatedBroadcastChannel] @@ -310,7 +312,7 @@ private class StateFlowImpl( initialState: Any // T | NULL ) : AbstractSharedFlow(), MutableStateFlow, CancellableFlow, FusibleFlow { private val _state = atomic(initialState) // T | NULL - private var sequence = 0 // serializes updates, value update is in process when sequence is odd + private val sequence = atomic(0) // serializes updates, value update is in process when sequence is odd @Suppress("UNCHECKED_CAST") public override var value: T @@ -322,19 +324,19 @@ private class StateFlowImpl( private fun updateState(expectedState: Any?, newState: Any): Boolean { var curSequence = 0 - var curSlots: Array? = this.slots // benign race, we will not use it + var curSlots: SharedFlowState? = slots // benign race, we will not use it synchronized(this) { val oldState = _state.value if (expectedState != null && oldState != expectedState) return false // CAS support if (oldState == newState) return true // Don't do anything if value is not changing, but CAS -> true _state.value = newState - curSequence = sequence + curSequence = sequence.value if (curSequence and 1 == 0) { // even sequence means quiescent state flow (no ongoing update) curSequence++ // make it odd - sequence = curSequence + sequence.value = curSequence } else { // update is already in process, notify it, and return - sequence = curSequence + 2 // change sequence to notify, keep it odd + sequence.value = curSequence + 2 // change sequence to notify, keep it odd return true // updated } curSlots = slots // read current reference to collectors under lock @@ -347,17 +349,21 @@ private class StateFlowImpl( */ while (true) { // Benign race on element read from array - curSlots?.forEach { - it?.makePending() + val _cs = curSlots + if (_cs != null) { + for (index in 0 until _cs.size) { + _cs[index]?.makePending() + } } + // check if the value was updated again while we were updating the old one synchronized(this) { - if (sequence == curSequence) { // nothing changed, we are done - sequence = curSequence + 1 // make sequence even again + if (sequence.value == curSequence) { // nothing changed, we are done + sequence.value = curSequence + 1 // make sequence even again return true // done, updated } // reread everything for the next loop under the lock - curSequence = sequence + curSequence = sequence.value curSlots = slots } } @@ -409,7 +415,6 @@ private class StateFlowImpl( } override fun createSlot() = StateFlowSlot() - override fun createSlotArray(size: Int): Array = arrayOfNulls(size) override fun fuse(context: CoroutineContext, capacity: Int, onBufferOverflow: BufferOverflow) = fuseStateFlow(context, capacity, onBufferOverflow) diff --git a/kotlinx-coroutines-core/common/src/flow/internal/AbstractSharedFlow.kt b/kotlinx-coroutines-core/common/src/flow/internal/AbstractSharedFlow.kt index 39ca98391f..385a530ba8 100644 --- a/kotlinx-coroutines-core/common/src/flow/internal/AbstractSharedFlow.kt +++ b/kotlinx-coroutines-core/common/src/flow/internal/AbstractSharedFlow.kt @@ -4,6 +4,7 @@ package kotlinx.coroutines.flow.internal +import kotlinx.atomicfu.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.flow.* import kotlinx.coroutines.internal.* @@ -20,41 +21,68 @@ internal abstract class AbstractSharedFlowSlot { abstract fun freeLocked(flow: F): Array?> // returns continuations to resume after lock } +internal class SharedFlowState( + val size: Int // size of array, see afu#149 +) { + private val arr: AtomicArray = atomicArrayOfNulls(size) + + operator fun set(index: Int, value: S?) { + arr[index].value = value + } + + operator fun get(index: Int): S? = arr[index].value + + fun getBufferAt(index: Long): Any? { + return get(index.toInt() and (size - 1)) + } + + fun setBufferAt(index: Long, item: S) { + arr.get(index.toInt() and (size - 1)).value = item + } + + fun copyInto(newSlots: SharedFlowState) { + for (i in 0 until size) { + newSlots.arr[i].value = get(i) + } + } +} + internal abstract class AbstractSharedFlow> : SynchronizedObject() { - @Suppress("UNCHECKED_CAST") - protected var slots: Array? = null // allocated when needed - private set - protected var nCollectors = 0 // number of allocated (!free) slots - private set - private var nextIndex = 0 // oracle for the next free slot index - private var _subscriptionCount: SubscriptionCountStateFlow? = null // init on first need + private val _slots = atomic?>(null) // allocated when needed + protected val slots: SharedFlowState? get() = _slots.value + private val _nCollectors = atomic(0) // number of allocated (!free) slots + protected val nCollectors: Int get() = _nCollectors.value // number of allocated (!free) slots + + private val nextIndex = atomic(0) // oracle for the next free slot index + private val _subscriptionCount = atomic(null) // init on first need val subscriptionCount: StateFlow get() = synchronized(this) { // allocate under lock in sync with nCollectors variable - _subscriptionCount ?: SubscriptionCountStateFlow(nCollectors).also { - _subscriptionCount = it + _subscriptionCount.value ?: SubscriptionCountStateFlow(nCollectors).also { + _subscriptionCount.value = it } } protected abstract fun createSlot(): S - protected abstract fun createSlotArray(size: Int): Array - @Suppress("UNCHECKED_CAST") protected fun allocateSlot(): S { // Actually create slot under lock var subscriptionCount: SubscriptionCountStateFlow? = null val slot = synchronized(this) { - val slots = when (val curSlots = slots) { - null -> createSlotArray(2).also { slots = it } + val slots = when (val curSlots = _slots.value) { + null -> SharedFlowState(2).also { _slots.value = it } else -> if (nCollectors >= curSlots.size) { - curSlots.copyOf(2 * curSlots.size).also { slots = it } + val newSlots = SharedFlowState(2 * curSlots.size) + curSlots.copyInto(newSlots) + _slots.value = newSlots + newSlots } else { curSlots } } - var index = nextIndex + var index = nextIndex.value var slot: S while (true) { slot = slots[index] ?: createSlot().also { slots[index] = it } @@ -62,9 +90,9 @@ internal abstract class AbstractSharedFlow> : Sync if (index >= slots.size) index = 0 if ((slot as AbstractSharedFlowSlot).allocateLocked(this)) break // break when found and allocated free slot } - nextIndex = index - nCollectors++ - subscriptionCount = _subscriptionCount // retrieve under lock if initialized + nextIndex.value = index + _nCollectors.incrementAndGet() + subscriptionCount = _subscriptionCount.value // retrieve under lock if initialized slot } // increments subscription count @@ -77,10 +105,10 @@ internal abstract class AbstractSharedFlow> : Sync // Release slot under lock var subscriptionCount: SubscriptionCountStateFlow? = null val resumes = synchronized(this) { - nCollectors-- - subscriptionCount = _subscriptionCount // retrieve under lock if initialized + _nCollectors.decrementAndGet() + subscriptionCount = _subscriptionCount.value // retrieve under lock if initialized // Reset next index oracle if we have no more active collectors for more predictable behavior next time - if (nCollectors == 0) nextIndex = 0 + if (_nCollectors.value == 0) nextIndex.value = 0 (slot as AbstractSharedFlowSlot).freeLocked(this) } /* @@ -94,9 +122,13 @@ internal abstract class AbstractSharedFlow> : Sync } protected inline fun forEachSlotLocked(block: (S) -> Unit) { - if (nCollectors == 0) return - slots?.forEach { slot -> - if (slot != null) block(slot) + if (_nCollectors.value == 0) return + val _slots = _slots.value + if (_slots != null) { + for (i in 0 until _slots.size) { + val slot = _slots.get(i) + if (slot != null) block(slot) + } } } } diff --git a/kotlinx-coroutines-core/common/src/flow/internal/FlowCoroutine.kt b/kotlinx-coroutines-core/common/src/flow/internal/FlowCoroutine.kt index 9a81eefa2d..c79fe643dd 100644 --- a/kotlinx-coroutines-core/common/src/flow/internal/FlowCoroutine.kt +++ b/kotlinx-coroutines-core/common/src/flow/internal/FlowCoroutine.kt @@ -54,7 +54,7 @@ internal fun scopedFlow(@BuilderInference block: suspend CoroutineScope.(Flo private class FlowCoroutine( context: CoroutineContext, uCont: Continuation -) : ScopeCoroutine(context, uCont) { +) : ScopeCoroutine(context, uCont, true) { override fun childCancelled(cause: Throwable): Boolean { if (cause is ChildCancelledException) return true return cancelImpl(cause) diff --git a/kotlinx-coroutines-core/common/src/flow/operators/Share.kt b/kotlinx-coroutines-core/common/src/flow/operators/Share.kt index 2b690e3c04..c38f1d1b33 100644 --- a/kotlinx-coroutines-core/common/src/flow/operators/Share.kt +++ b/kotlinx-coroutines-core/common/src/flow/operators/Share.kt @@ -10,6 +10,7 @@ package kotlinx.coroutines.flow import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.flow.internal.* +import kotlinx.coroutines.internal.* import kotlin.coroutines.* import kotlin.jvm.* @@ -204,8 +205,9 @@ private fun CoroutineScope.launchSharing( * * Delayed sharing strategies have a chance to immediately observe consecutive subscriptions. * E.g. in the cases like `flow.shareIn(...); flow.take(1)` we want sharing strategy to see the initial subscription * * Eager sharing does not start immediately, so the subscribers have actual chance to subscribe _prior_ to sharing. + * // TODO: kludge: native-mt doesn't support undispatched */ - val start = if (started == SharingStarted.Eagerly) CoroutineStart.DEFAULT else CoroutineStart.UNDISPATCHED + val start = if (started == SharingStarted.Eagerly || isNativeMt) CoroutineStart.DEFAULT else CoroutineStart.UNDISPATCHED return launch(context, start = start) { // the single coroutine to rule the sharing // Optimize common built-in started strategies when { diff --git a/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt b/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt index fb254a0ebc..c8ebf4de8f 100644 --- a/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt +++ b/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt @@ -16,11 +16,4 @@ internal typealias SubscribersList = MutableList "and K/JS platforms and it is unsafe to use it anywhere else") internal expect fun subscriberList(): SubscribersList -internal expect class ReentrantLock() { - fun tryLock(): Boolean - fun unlock(): Unit -} - -internal expect inline fun ReentrantLock.withLock(action: () -> T): T - internal expect fun identitySet(expectedSize: Int): MutableSet diff --git a/kotlinx-coroutines-core/common/src/internal/DispatchedTask.kt b/kotlinx-coroutines-core/common/src/internal/DispatchedTask.kt index d982f95bdf..9c4d1217ff 100644 --- a/kotlinx-coroutines-core/common/src/internal/DispatchedTask.kt +++ b/kotlinx-coroutines-core/common/src/internal/DispatchedTask.kt @@ -83,7 +83,7 @@ internal abstract class DispatchedTask( val taskContext = this.taskContext var fatalException: Throwable? = null try { - val delegate = delegate as DispatchedContinuation + val delegate = delegate.useLocal() as DispatchedContinuation // cast must succeed val continuation = delegate.continuation withContinuationContext(continuation, delegate.countOrElement) { val context = continuation.context @@ -147,14 +147,15 @@ internal abstract class DispatchedTask( } } -internal fun DispatchedTask.dispatch(mode: Int) { +internal fun CancellableContinuationImpl.dispatch(mode: Int) { assert { mode != MODE_UNINITIALIZED } // invalid mode value for this method val delegate = this.delegate + val local = delegate.asLocalOrNull() // go to shareableResume when called from wrong worker val undispatched = mode == MODE_UNDISPATCHED - if (!undispatched && delegate is DispatchedContinuation<*> && mode.isCancellableMode == resumeMode.isCancellableMode) { + if (!undispatched && local is DispatchedContinuation<*> && mode.isCancellableMode == resumeMode.isCancellableMode) { // dispatch directly using this instance's Runnable implementation - val dispatcher = delegate.dispatcher - val context = delegate.context + val dispatcher = local.dispatcher + val context = local.context if (dispatcher.isDispatchNeeded(context)) { dispatcher.dispatch(context, this) } else { @@ -163,12 +164,12 @@ internal fun DispatchedTask.dispatch(mode: Int) { } else { // delegate is coming from 3rd-party interceptor implementation (and does not support cancellation) // or undispatched mode was requested - resume(delegate, undispatched) + shareableResume(delegate, undispatched) } } @Suppress("UNCHECKED_CAST") -internal fun DispatchedTask.resume(delegate: Continuation, undispatched: Boolean) { +internal fun CancellableContinuationImpl.resume(delegate: Continuation, undispatched: Boolean) { // This resume is never cancellable. The result is always delivered to delegate continuation. val state = takeState() val exception = getExceptionalResult(state) @@ -179,7 +180,7 @@ internal fun DispatchedTask.resume(delegate: Continuation, undispatche } } -private fun DispatchedTask<*>.resumeUnconfined() { +private fun CancellableContinuationImpl.resumeUnconfined() { val eventLoop = ThreadLocalEventLoop.eventLoop if (eventLoop.isUnconfinedLoopActive) { // When unconfined loop is active -- dispatch continuation for execution to avoid stack overflow @@ -187,7 +188,7 @@ private fun DispatchedTask<*>.resumeUnconfined() { } else { // Was not active -- run event loop until all unconfined tasks are executed runUnconfinedEventLoop(eventLoop) { - resume(delegate, undispatched = true) + resume(delegate.useLocal(), undispatched = true) } } } diff --git a/kotlinx-coroutines-core/common/src/internal/LockFreeLinkedList.common.kt b/kotlinx-coroutines-core/common/src/internal/LockFreeLinkedList.common.kt index 8b20ade1f0..f65f134145 100644 --- a/kotlinx-coroutines-core/common/src/internal/LockFreeLinkedList.common.kt +++ b/kotlinx-coroutines-core/common/src/internal/LockFreeLinkedList.common.kt @@ -69,7 +69,7 @@ public expect open class RemoveFirstDesc(queue: LockFreeLinkedListNode): Abst public expect abstract class AbstractAtomicDesc : AtomicDesc { final override fun prepare(op: AtomicOp<*>): Any? final override fun complete(op: AtomicOp<*>, failure: Any?) - protected open fun failure(affected: LockFreeLinkedListNode): Any? + protected open fun failure(affected: LockFreeLinkedListNode?): Any? // must fail on null for unlinked nodes on K/N protected open fun retry(affected: LockFreeLinkedListNode, next: Any): Boolean public abstract fun finishPrepare(prepareOp: PrepareOp) // non-null on failure public open fun onPrepare(prepareOp: PrepareOp): Any? // non-null on failure diff --git a/kotlinx-coroutines-core/common/src/internal/ManualMemoryManagement.common.kt b/kotlinx-coroutines-core/common/src/internal/ManualMemoryManagement.common.kt new file mode 100644 index 0000000000..7bb2e532d9 --- /dev/null +++ b/kotlinx-coroutines-core/common/src/internal/ManualMemoryManagement.common.kt @@ -0,0 +1,8 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.internal + +internal expect inline fun disposeLockFreeLinkedList(list: () -> LockFreeLinkedListNode?) // only needed on Kotlin/Native +internal expect inline fun storeCyclicRef(block: () -> Unit) // nop on native diff --git a/kotlinx-coroutines-core/common/src/internal/Scopes.kt b/kotlinx-coroutines-core/common/src/internal/Scopes.kt index ad8d86ed5a..c970b5d1f0 100644 --- a/kotlinx-coroutines-core/common/src/internal/Scopes.kt +++ b/kotlinx-coroutines-core/common/src/internal/Scopes.kt @@ -6,7 +6,6 @@ package kotlinx.coroutines.internal import kotlinx.coroutines.* import kotlin.coroutines.* -import kotlin.coroutines.intrinsics.* import kotlin.jvm.* /** @@ -14,18 +13,29 @@ import kotlin.jvm.* */ internal open class ScopeCoroutine( context: CoroutineContext, - @JvmField val uCont: Continuation // unintercepted continuation + uCont: Continuation, + useInitNativeKludge: Boolean ) : AbstractCoroutine(context, true, true), CoroutineStackFrame { - - final override val callerFrame: CoroutineStackFrame? get() = uCont as? CoroutineStackFrame + @JvmField + val uCont: Continuation = uCont.asShareable() // unintercepted continuation, shareable + final override val callerFrame: CoroutineStackFrame? get() = uCont.asLocal() as? CoroutineStackFrame final override fun getStackTraceElement(): StackTraceElement? = null final override val isScopedCoroutine: Boolean get() = true internal val parent: Job? get() = parentHandle?.parent + init { + // Kludge for native + if (useInitNativeKludge && !isReuseSupportedInPlatform()) initParentForNativeUndispatchedCoroutine() + } + + protected open fun initParentForNativeUndispatchedCoroutine() { + initParentJob(parentContext[Job]) + } + override fun afterCompletion(state: Any?) { // Resume in a cancellable way by default when resuming from another context - uCont.intercepted().resumeCancellableWith(recoverResult(state, uCont)) + uCont.shareableInterceptedResumeCancellableWith(recoverResult(state, uCont)) } override fun afterResume(state: Any?) { diff --git a/kotlinx-coroutines-core/common/src/internal/Sharing.common.kt b/kotlinx-coroutines-core/common/src/internal/Sharing.common.kt new file mode 100644 index 0000000000..dfe3b8fcd6 --- /dev/null +++ b/kotlinx-coroutines-core/common/src/internal/Sharing.common.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.internal + +import kotlinx.coroutines.* +import kotlin.coroutines.* + +internal expect open class ShareableRefHolder() +internal expect fun ShareableRefHolder.disposeSharedRef() +internal expect fun T.asShareable(): DisposableHandle where T : DisposableHandle, T : ShareableRefHolder +internal expect fun CoroutineDispatcher.asShareable(): CoroutineDispatcher +internal expect fun Continuation.asShareable() : Continuation +internal expect fun Continuation.asLocal() : Continuation +internal expect fun Continuation.asLocalOrNull() : Continuation? +internal expect fun Continuation.asLocalOrNullIfNotUsed() : Continuation? +internal expect fun Continuation.useLocal() : Continuation +internal expect fun Continuation.shareableInterceptedResumeCancellableWith(result: Result) +internal expect fun Continuation.shareableInterceptedResumeWith(result: Result) +internal expect fun disposeContinuation(cont: () -> Continuation<*>) +internal expect fun CancellableContinuationImpl.shareableResume(delegate: Continuation, undispatched: Boolean) + +internal expect fun (suspend (T) -> R).asShareable(): suspend (T) -> R + +internal expect fun isReuseSupportedInPlatform(): Boolean +internal expect fun ArrayList.addOrUpdate(element: T, update: (ArrayList) -> Unit) +internal expect fun ArrayList.addOrUpdate(index: Int, element: T, update: (ArrayList) -> Unit) +internal expect fun Any.weakRef(): Any +internal expect fun Any?.unweakRef(): Any? diff --git a/kotlinx-coroutines-core/common/src/internal/Synchronized.common.kt b/kotlinx-coroutines-core/common/src/internal/Synchronized.common.kt index 059b234d8f..5b6c71e417 100644 --- a/kotlinx-coroutines-core/common/src/internal/Synchronized.common.kt +++ b/kotlinx-coroutines-core/common/src/internal/Synchronized.common.kt @@ -17,3 +17,5 @@ public expect open class SynchronizedObject() // marker abstract class */ @InternalCoroutinesApi public expect inline fun synchronized(lock: SynchronizedObject, block: () -> T): T + +internal expect val isNativeMt: Boolean diff --git a/kotlinx-coroutines-core/common/src/internal/ThreadSafeHeap.kt b/kotlinx-coroutines-core/common/src/internal/ThreadSafeHeap.kt index 2100454be3..8193ada8fd 100644 --- a/kotlinx-coroutines-core/common/src/internal/ThreadSafeHeap.kt +++ b/kotlinx-coroutines-core/common/src/internal/ThreadSafeHeap.kt @@ -5,6 +5,7 @@ package kotlinx.coroutines.internal import kotlinx.atomicfu.* +import kotlinx.atomicfu.locks.* import kotlinx.coroutines.* /** diff --git a/kotlinx-coroutines-core/common/src/intrinsics/Undispatched.kt b/kotlinx-coroutines-core/common/src/intrinsics/Undispatched.kt index 38e870ef9c..da9b6460eb 100644 --- a/kotlinx-coroutines-core/common/src/intrinsics/Undispatched.kt +++ b/kotlinx-coroutines-core/common/src/intrinsics/Undispatched.kt @@ -125,6 +125,8 @@ private inline fun ScopeCoroutine.undispatchedResult( if (result === COROUTINE_SUSPENDED) return COROUTINE_SUSPENDED // (1) val state = makeCompletingOnce(result) if (state === COMPLETING_WAITING_CHILDREN) return COROUTINE_SUSPENDED // (2) + // When scope coroutine does not suspend on Kotlin/Native it shall dispose its continuation which it will not use + disposeContinuation { uCont } return if (state is CompletedExceptionally) { // (3) when { shouldThrow(state.cause) -> throw recoverStackTrace(state.cause, uCont) diff --git a/kotlinx-coroutines-core/common/src/selects/Select.kt b/kotlinx-coroutines-core/common/src/selects/Select.kt index 921322244a..ecbdecc2e3 100644 --- a/kotlinx-coroutines-core/common/src/selects/Select.kt +++ b/kotlinx-coroutines-core/common/src/selects/Select.kt @@ -233,10 +233,12 @@ private val selectOpSequenceNumber = SeqNumber() @PublishedApi internal class SelectBuilderImpl( - private val uCont: Continuation // unintercepted delegate continuation + uCont: Continuation ) : LockFreeLinkedListHead(), SelectBuilder, SelectInstance, Continuation, CoroutineStackFrame { + private val uCont: Continuation = uCont.asShareable() // unintercepted delegate continuation, shareable + override val callerFrame: CoroutineStackFrame? get() = uCont as? CoroutineStackFrame @@ -305,7 +307,7 @@ internal class SelectBuilderImpl( // Resumes in dispatched way so that it can be called from an arbitrary context override fun resumeSelectWithException(exception: Throwable) { doResume({ CompletedExceptionally(recoverStackTrace(exception, uCont)) }) { - uCont.intercepted().resumeWith(Result.failure(exception)) + uCont.shareableInterceptedResumeWith(Result.failure(exception)) } } @@ -345,16 +347,19 @@ internal class SelectBuilderImpl( internal fun handleBuilderException(e: Throwable) { if (trySelect()) { resumeWithException(e) - } else if (e !is CancellationException) { - /* - * Cannot handle this exception -- builder was already resumed with a different exception, - * so treat it as "unhandled exception". But only if it is not the completion reason - * and it's not the cancellation. Otherwise, in the face of structured concurrency - * the same exception will be reported to the global exception handler. - */ - val result = getResult() - if (result !is CompletedExceptionally || unwrap(result.cause) !== unwrap(e)) { - handleCoroutineException(context, e) + } else { + disposeLockFreeLinkedList { this } + if (e !is CancellationException) { + /* + * Cannot handle this exception -- builder was already resumed with a different exception, + * so treat it as "unhandled exception". But only if it is not the completion reason + * and it's not the cancellation. Otherwise, in the face of structured concurrency + * the same exception will be reported to the global exception handler. + */ + val result = getResult() + if (result !is CompletedExceptionally || unwrap(result.cause) !== unwrap(e)) { + handleCoroutineException(context, e) + } } } } @@ -380,10 +385,12 @@ internal class SelectBuilderImpl( } private fun doAfterSelect() { + val parentHandle = _parentHandle.getAndSet(null) parentHandle?.dispose() forEach { - it.handle.dispose() + it.dispose() } + disposeLockFreeLinkedList { this } } override fun trySelect(): Boolean { @@ -652,6 +659,13 @@ internal class SelectBuilderImpl( } private class DisposeNode( - @JvmField val handle: DisposableHandle - ) : LockFreeLinkedListNode() + handle: DisposableHandle + ) : LockFreeLinkedListNode() { + private val _handle = atomic(handle) + + fun dispose() { + val handle = _handle.getAndSet(null) + handle?.dispose() + } + } } diff --git a/kotlinx-coroutines-core/common/src/sync/Mutex.kt b/kotlinx-coroutines-core/common/src/sync/Mutex.kt index 681d5db6b0..d481af3907 100644 --- a/kotlinx-coroutines-core/common/src/sync/Mutex.kt +++ b/kotlinx-coroutines-core/common/src/sync/Mutex.kt @@ -361,16 +361,27 @@ internal class MutexImpl(locked: Boolean) : Mutex, SelectClause2 { } private class LockedQueue( - @Volatile @JvmField var owner: Any + owner: Any ) : LockFreeLinkedListHead() { + private val _owner = atomic(owner) + var owner: Any + get() = _owner.value + set(value) { _owner.value = value } + override fun toString(): String = "LockedQueue[$owner]" } private abstract inner class LockWaiter( - @JvmField val owner: Any? + owner: Any? ) : LockFreeLinkedListNode(), DisposableHandle { + private val _owner = atomic(owner) + var owner: Any? + get() = _owner.value + set(value) { _owner.value = value } + private val isTaken = atomic(false) fun take(): Boolean = isTaken.compareAndSet(false, true) + final override fun dispose() { remove() } abstract fun tryResumeLockWaiter(): Boolean abstract fun completeResumeLockWaiter() @@ -400,10 +411,7 @@ internal class MutexImpl(locked: Boolean) : Mutex, SelectClause2 { ) : LockWaiter(owner) { override fun tryResumeLockWaiter(): Boolean = take() && select.trySelect() override fun completeResumeLockWaiter() { - block.startCoroutineCancellable(receiver = this@MutexImpl, completion = select.completion) { - // if this continuation gets cancelled during dispatch to the caller, then release the lock - unlock(owner) - } + startCoroutine(CoroutineStart.DEFAULT, this@MutexImpl, select.completion, { unlock(owner) }, block) } override fun toString(): String = "LockSelect[$owner, $select] for ${this@MutexImpl}" } diff --git a/kotlinx-coroutines-core/common/test/CoroutineScopeTest.kt b/kotlinx-coroutines-core/common/test/CoroutineScopeTest.kt index c46f41a073..e9a92d2317 100644 --- a/kotlinx-coroutines-core/common/test/CoroutineScopeTest.kt +++ b/kotlinx-coroutines-core/common/test/CoroutineScopeTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ @file:Suppress("UNREACHABLE_CODE") @@ -104,13 +104,17 @@ class CoroutineScopeTest : TestBase() { launch { expect(3) try { + println("1") delay(Long.MAX_VALUE) } finally { + println("F") expect(4) } } + println("E?") expect(2) + println("3") delay(Long.MAX_VALUE) 42 } @@ -118,9 +122,11 @@ class CoroutineScopeTest : TestBase() { val outerJob = launch(NonCancellable) { expect(1) try { + println("2") callJobScoped() expectUnreached() } catch (e: JobCancellationException) { + println("4") expect(5) if (RECOVER_STACK_TRACES) { val cause = e.cause as JobCancellationException // shall be recovered JCE @@ -131,8 +137,11 @@ class CoroutineScopeTest : TestBase() { } } repeat(3) { yield() } // let everything to start properly + println("C") outerJob.cancel() + println("J") outerJob.join() + println("D") finish(6) } diff --git a/kotlinx-coroutines-core/common/test/LimitedParallelismSharedTest.kt b/kotlinx-coroutines-core/common/test/LimitedParallelismSharedTest.kt index d01e85716b..ca4477b037 100644 --- a/kotlinx-coroutines-core/common/test/LimitedParallelismSharedTest.kt +++ b/kotlinx-coroutines-core/common/test/LimitedParallelismSharedTest.kt @@ -4,12 +4,14 @@ package kotlinx.coroutines +import kotlinx.coroutines.internal.* import kotlin.test.* class LimitedParallelismSharedTest : TestBase() { @Test fun testLimitedDefault() = runTest { + if (!isReuseSupportedInPlatform()) return@runTest // Test that evaluates the very basic completion of tasks in limited dispatcher // for all supported platforms. // For more specific and concurrent tests, see 'concurrent' package. diff --git a/kotlinx-coroutines-core/common/test/flow/operators/BackgroundFlowTest.kt b/kotlinx-coroutines-core/common/test/flow/operators/BackgroundFlowTest.kt new file mode 100644 index 0000000000..46ac2b7cb8 --- /dev/null +++ b/kotlinx-coroutines-core/common/test/flow/operators/BackgroundFlowTest.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.flow.operators + +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import kotlin.test.* + +/** + * Test some flow operators in background thread, + * mostly for the purpose of checking Kotlin/Native implementation. + */ +class BackgroundFlowTest : TestBase() { + @Test + fun testFlowCombine() = runTest { + withContext(Dispatchers.Default) { + val flow = flowOf(1) + val combo = combine(flow, flow) { a, b -> a + b } + assertEquals(2, combo.first()) + } + } +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/common/test/flow/sharing/ShareInTest.kt b/kotlinx-coroutines-core/common/test/flow/sharing/ShareInTest.kt index cf83a50b0f..e7a4eaa61d 100644 --- a/kotlinx-coroutines-core/common/test/flow/sharing/ShareInTest.kt +++ b/kotlinx-coroutines-core/common/test/flow/sharing/ShareInTest.kt @@ -6,6 +6,7 @@ package kotlinx.coroutines.flow import kotlinx.coroutines.* import kotlinx.coroutines.channels.* +import kotlinx.coroutines.internal.* import kotlin.test.* class ShareInTest : TestBase() { @@ -213,6 +214,7 @@ class ShareInTest : TestBase() { @Test fun testShouldStart() = runTest { + if (isNativeMt) return@runTest val flow = flow { expect(2) emit(1) @@ -229,6 +231,7 @@ class ShareInTest : TestBase() { @Test fun testShouldStartScalar() = runTest { + if (isNativeMt) return@runTest val j = Job() val shared = flowOf(239).stateIn(this + j, SharingStarted.Lazily, 42) assertEquals(42, shared.first()) diff --git a/kotlinx-coroutines-core/concurrent/src/Builders.concurrent.kt b/kotlinx-coroutines-core/concurrent/src/Builders.concurrent.kt index 8a6c09231b..656bc87109 100644 --- a/kotlinx-coroutines-core/concurrent/src/Builders.concurrent.kt +++ b/kotlinx-coroutines-core/concurrent/src/Builders.concurrent.kt @@ -11,4 +11,4 @@ import kotlin.coroutines.* * This function should not be used from a coroutine. It is designed to bridge regular blocking code * to libraries that are written in suspending style, to be used in `main` functions and in tests. */ -public expect fun runBlocking(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> T): T \ No newline at end of file +public expect fun runBlocking(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> T): T diff --git a/kotlinx-coroutines-core/concurrent/src/MultithreadedDispatchers.common.kt b/kotlinx-coroutines-core/concurrent/src/MultithreadedDispatchers.common.kt index a2b4241f0f..f05f0c0d82 100644 --- a/kotlinx-coroutines-core/concurrent/src/MultithreadedDispatchers.common.kt +++ b/kotlinx-coroutines-core/concurrent/src/MultithreadedDispatchers.common.kt @@ -6,6 +6,3 @@ package kotlinx.coroutines @ExperimentalCoroutinesApi public expect fun newSingleThreadContext(name: String): CloseableCoroutineDispatcher - -@ExperimentalCoroutinesApi -public expect fun newFixedThreadPoolContext(nThreads: Int, name: String): CloseableCoroutineDispatcher diff --git a/kotlinx-coroutines-core/concurrent/src/internal/LockFreeLinkedList.kt b/kotlinx-coroutines-core/concurrent/src/internal/LockFreeLinkedList.kt index b4b36dad34..ff5492c3f6 100644 --- a/kotlinx-coroutines-core/concurrent/src/internal/LockFreeLinkedList.kt +++ b/kotlinx-coroutines-core/concurrent/src/internal/LockFreeLinkedList.kt @@ -63,18 +63,25 @@ public actual typealias PrepareOp = LockFreeLinkedListNode.PrepareOp @Suppress("LeakingThis") @InternalCoroutinesApi public actual open class LockFreeLinkedListNode { - private val _next = atomic(this) // Node | Removed | OpDescriptor - private val _prev = atomic(this) // Node to the left (cannot be marked as removed) + // those _next & _prev refs can be null on Kotlin/Native when doubly-linked list is unlinked + private val _next = atomic(this) // Node | Removed | OpDescriptor + private val _prev = atomic(this) // Node to the left (cannot be marked as removed) private val _removedRef = atomic(null) // lazily cached removed ref to this private fun removed(): Removed = - _removedRef.value ?: Removed(this).also { _removedRef.lazySet(it) } + _removedRef.value ?: Removed(this).also { + storeCyclicRef { _removedRef.lazySet(it) } + } @PublishedApi internal abstract class CondAddOp( @JvmField val newNode: Node ) : AtomicOp() { - @JvmField var oldNext: Node? = null + private val _oldNext = atomic(null) + + var oldNext: Node? + get() = _oldNext.value + set(value) { _oldNext.value = value } override fun complete(affected: Node, failure: Any?) { val success = failure == null @@ -95,7 +102,8 @@ public actual open class LockFreeLinkedListNode { public actual open val isRemoved: Boolean get() = next is Removed // LINEARIZABLE. Returns Node | Removed - public val next: Any get() { + // Returns null for the unlinked node on Kotlin/Native + public val next: Any? get() { _next.loop { next -> if (next !is OpDescriptor) return next next.perform(this) @@ -103,19 +111,26 @@ public actual open class LockFreeLinkedListNode { } // LINEARIZABLE. Returns next non-removed Node - public actual val nextNode: Node get() = next.unwrap() + public actual val nextNode: Node get() = next?.unwrap() ?: this // it could have been unlinked on Kotlin/Native // LINEARIZABLE WHEN THIS NODE IS NOT REMOVED: // Returns prev non-removed Node, makes sure prev is correct (prev.next === this) // NOTE: if this node is removed, then returns non-removed previous node without applying // prev.next correction, which does not provide linearizable backwards iteration, but can be used to // resume forward iteration when current node was removed. - public actual val prevNode: Node - get() = correctPrev(null) ?: findPrevNonRemoved(_prev.value) + // NOTE: It could have been unlinked on Kotlin/Native. In this case `this` is returned. + public actual val prevNode: Node get() = prevOrNull ?: this + + // Just line prevNode, but return nulls for unlinked nodes on Kotlin/Native + @PublishedApi + internal val prevOrNull: Node? + get() { + return correctPrev(null) ?: findPrevNonRemoved(_prev.value ?: return null) + } - private tailrec fun findPrevNonRemoved(current: Node): Node { + private tailrec fun findPrevNonRemoved(current: Node): Node? { if (!current.isRemoved) return current - return findPrevNonRemoved(current._prev.value) + return findPrevNonRemoved(current._prev.value ?: return null) } // ------ addOneIfEmpty ------ @@ -141,7 +156,8 @@ public actual open class LockFreeLinkedListNode { */ public actual fun addLast(node: Node) { while (true) { // lock-free loop on prev.next - if (prevNode.addNext(node, this)) return + val prev = prevOrNull ?: return // can be unlinked on Kotlin/Native + if (prev.addNext(node, this)) return } } @@ -153,7 +169,8 @@ public actual open class LockFreeLinkedListNode { public actual inline fun addLastIf(node: Node, crossinline condition: () -> Boolean): Boolean { val condAdd = makeCondAddOp(node, condition) while (true) { // lock-free loop on prev.next - val prev = prevNode // sentinel node is never removed, so prev is always defined + // sentinel node is never removed, so prev is always defined, but can be concurrently unlinked on Kotlin/Native + val prev = prevOrNull ?: return false when (prev.tryCondAddNext(node, this, condAdd)) { SUCCESS -> return true FAILURE -> return false @@ -163,7 +180,8 @@ public actual open class LockFreeLinkedListNode { public actual inline fun addLastIfPrev(node: Node, predicate: (Node) -> Boolean): Boolean { while (true) { // lock-free loop on prev.next - val prev = prevNode // sentinel node is never removed, so prev is always defined + // sentinel node is never removed, so prev is always defined, but can be unlinked on Kotlin/Native + val prev = prevOrNull ?: return false if (!predicate(prev)) return false if (prev.addNext(node, this)) return true } @@ -176,7 +194,8 @@ public actual open class LockFreeLinkedListNode { ): Boolean { val condAdd = makeCondAddOp(node, condition) while (true) { // lock-free loop on prev.next - val prev = prevNode // sentinel node is never removed, so prev is always defined + // sentinel node is never removed, so prev is always defined, but can be unlinked on Kotlin/Native + val prev = prevOrNull ?: return false if (!predicate(prev)) return false when (prev.tryCondAddNext(node, this, condAdd)) { SUCCESS -> return true @@ -244,13 +263,16 @@ public actual open class LockFreeLinkedListNode { public actual open fun remove(): Boolean = removeOrNext() == null - // returns null if removed successfully or next node if this node is already removed + // returns null if removed successfully or next node if this node is already removed or unlinked on Kotlin/Native @PublishedApi internal fun removeOrNext(): Node? { while (true) { // lock-free loop on next - val next = this.next + val next = this.next ?: return null // abort when unlinked on Kotlin/Native if (next is Removed) return next.ref // was already removed -- don't try to help (original thread will take care) - if (next === this) return next // was not even added + if (next === this) { + _prev.value = null // Unlink this node from itself to avoid cycle on Kotlin/Native + return next // was not even added + } val removed = (next as Node).removed() if (_next.compareAndSet(next, removed)) { // was removed successfully (linearized remove) -- fixup the list @@ -263,7 +285,8 @@ public actual open class LockFreeLinkedListNode { // Helps with removal of this node public actual fun helpRemove() { // Note: this node must be already removed - (next as Removed).ref.helpRemovePrev() + // (next as Removed?)?.ref?.correctPrev(null) + (next as Removed).ref?.helpRemovePrev() } // Helps with removal of nodes that are previous to this @@ -273,9 +296,9 @@ public actual open class LockFreeLinkedListNode { // called on a removed node. There's always at least one non-removed node (list head). var node = this while (true) { - val next = node.next + val next = node.next ?: return // abort when unlinked on Kotlin/Native if (next !is Removed) break - node = next.ref + node = next.ref ?: return // abort when unlinked on Kotlin/Native } // Found a non-removed node node.correctPrev(null) @@ -375,12 +398,13 @@ public actual open class LockFreeLinkedListNode { final override val originalNext: Node? get() = _originalNext.value // check node predicates here, must signal failure if affect is not of type T - protected override fun failure(affected: Node): Any? = - if (affected === queue) LIST_EMPTY else null + protected override fun failure(affected: Node?): Any? = + if (affected === queue || affected == null) LIST_EMPTY else null // must fail on null for unlinked nodes on K/N final override fun retry(affected: Node, next: Any): Boolean { if (next !is Removed) return false - next.ref.helpRemovePrev() // must help delete to ensure lock-freedom + val nextRef = next.ref ?: return false // abort when unlinked on Kotlin/Native + nextRef.helpRemovePrev() // must help delete to ensure lock-freedom return true } @@ -453,7 +477,7 @@ public actual open class LockFreeLinkedListNode { protected abstract val affectedNode: Node? protected abstract val originalNext: Node? protected open fun takeAffectedNode(op: OpDescriptor): Node? = affectedNode!! // null for RETRY_ATOMIC - protected open fun failure(affected: Node): Any? = null // next: Node | Removed + protected open fun failure(affected: Node?): Any? = null // next: Node | Removed // must fail on null for unlinked nodes on K/N protected open fun retry(affected: Node, next: Any): Boolean = false // next: Node | Removed protected abstract fun finishOnSuccess(affected: Node, next: Node) @@ -486,9 +510,11 @@ public actual open class LockFreeLinkedListNode { next.perform(affected) continue // and retry } + // on Kotlin/Native next can be already unlinked // next: Node | Removed - val failure = failure(affected) + val failure = failure(affected) // must fail on null for unlinked nodes on K/N if (failure != null) return failure // signal failure + next as Any // should have failed if was null if (retry(affected, next)) continue // retry operation val prepareOp = PrepareOp(affected, next as Node, this) if (affected._next.compareAndSet(next, prepareOp)) { @@ -560,7 +586,7 @@ public actual open class LockFreeLinkedListNode { * Returns the corrected value of the previous node while also correcting the `prev` pointer * (so that `this.prev.next === this`) and helps complete node removals to the left ot this node. * - * It returns `null` in two special cases: + * It returns `null` in special cases: * * * When this node is removed. In this case there is no need to waste time on corrections, because * remover of this node will ultimately call [correctPrev] on the next node and that will fix all @@ -568,13 +594,14 @@ public actual open class LockFreeLinkedListNode { * * When [op] descriptor is not `null` and operation descriptor that is [OpDescriptor.isEarlierThan] * that current [op] is found while traversing the list. This `null` result will be translated * by callers to [RETRY_ATOMIC]. + * * When list is unlinked on Kotlin/Native. */ private tailrec fun correctPrev(op: OpDescriptor?): Node? { val oldPrev = _prev.value - var prev: Node = oldPrev + var prev: Node = oldPrev ?: return null // abort when unlinked on Kotlin/Native var last: Node? = null // will be set so that last.next === prev while (true) { // move the left until first non-removed node - val prevNext: Any = prev._next.value + val prevNext: Any = prev._next.value ?: return null // abort when unlinked on Kotlin/Native when { // fast path to find quickly find prev node when everything is properly linked prevNext === this -> { @@ -604,7 +631,7 @@ public actual open class LockFreeLinkedListNode { prev = last last = null } else { - prev = prev._prev.value + prev = prev._prev.value ?: return null // abort when unlinked on Kotlin/Native } } else -> { // prevNext is a regular node, but not this -- help delete @@ -620,10 +647,20 @@ public actual open class LockFreeLinkedListNode { assert { next === this._next.value } } + /** + * Only needed on Kotlin/Native to unlink cyclic data structure. See [disposeLockFreeLinkedList]. + */ + internal fun unlinkRefs(last: Boolean) { + if (last) _next.value = null + _prev.value = null + } + override fun toString(): String = "${this::classSimpleName}@${this.hexAddress}" } -private class Removed(@JvmField val ref: Node) { +private class Removed(ref: Node) { + private val wRef: Any = ref.weakRef() + val ref: Node? get() = wRef.unweakRef() as Node? override fun toString(): String = "Removed[$ref]" } @@ -642,10 +679,10 @@ public actual open class LockFreeLinkedListHead : LockFreeLinkedListNode() { * Iterates over all elements in this list of a specified type. */ public actual inline fun forEach(block: (T) -> Unit) { - var cur: Node = next as Node - while (cur != this) { + var cur: Node? = next as Node? + while (cur != this && cur != null) { if (cur is T) block(cur) - cur = cur.nextNode + cur = cur.next?.unwrap() } } diff --git a/kotlinx-coroutines-core/concurrent/test/LimitedParallelismConcurrentTest.kt b/kotlinx-coroutines-core/concurrent/test/LimitedParallelismConcurrentTest.kt deleted file mode 100644 index 8d38f05b4b..0000000000 --- a/kotlinx-coroutines-core/concurrent/test/LimitedParallelismConcurrentTest.kt +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2016-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines - -import kotlinx.atomicfu.* -import kotlinx.coroutines.* -import kotlinx.coroutines.exceptions.* -import kotlin.test.* - -class LimitedParallelismConcurrentTest : TestBase() { - - private val targetParallelism = 4 - private val iterations = 100_000 - private val parallelism = atomic(0) - - private fun checkParallelism() { - val value = parallelism.incrementAndGet() - randomWait() - assertTrue { value <= targetParallelism } - parallelism.decrementAndGet() - } - - @Test - fun testLimitedExecutor() = runMtTest { - val executor = newFixedThreadPoolContext(targetParallelism, "test") - val view = executor.limitedParallelism(targetParallelism) - doStress { - repeat(iterations) { - launch(view) { - checkParallelism() - } - } - } - executor.close() - } - - private suspend inline fun doStress(crossinline block: suspend CoroutineScope.() -> Unit) { - repeat(stressTestMultiplier) { - coroutineScope { - block() - } - } - } - - @Test - fun testTaskFairness() = runMtTest { - val executor = newSingleThreadContext("test") - val view = executor.limitedParallelism(1) - val view2 = executor.limitedParallelism(1) - val j1 = launch(view) { - while (true) { - yield() - } - } - val j2 = launch(view2) { j1.cancel() } - joinAll(j1, j2) - executor.close() - } -} diff --git a/kotlinx-coroutines-core/concurrent/test/RunBlockingTest.kt b/kotlinx-coroutines-core/concurrent/test/RunBlockingTest.kt index 70f6b8ba60..cb23d7c752 100644 --- a/kotlinx-coroutines-core/concurrent/test/RunBlockingTest.kt +++ b/kotlinx-coroutines-core/concurrent/test/RunBlockingTest.kt @@ -10,6 +10,7 @@ import kotlin.test.* class RunBlockingTest : TestBase() { @Test + @Ignore fun testWithTimeoutBusyWait() = runMtTest { val value = withTimeoutOrNull(10) { while (isActive) { @@ -21,36 +22,6 @@ class RunBlockingTest : TestBase() { assertEquals("value", value) } - @Test - fun testPrivateEventLoop() { - expect(1) - runBlocking { - expect(2) - assertTrue(coroutineContext[ContinuationInterceptor] is EventLoop) - yield() // is supported! - expect(3) - } - finish(4) - } - - @Test - fun testOuterEventLoop() { - expect(1) - runBlocking { - expect(2) - val outerEventLoop = coroutineContext[ContinuationInterceptor] as EventLoop - runBlocking(coroutineContext) { - expect(3) - // still same event loop - assertSame(coroutineContext[ContinuationInterceptor], outerEventLoop) - yield() // still works - expect(4) - } - expect(5) - } - finish(6) - } - @Test fun testOtherDispatcher() = runMtTest { expect(1) @@ -67,23 +38,6 @@ class RunBlockingTest : TestBase() { thread.close() } - @Test - fun testCancellation() = runMtTest { - newFixedThreadPoolContext(2, "testCancellation").use { - val job = GlobalScope.launch(it) { - runBlocking(coroutineContext) { - while (true) { - yield() - } - } - } - - runBlocking { - job.cancelAndJoin() - } - } - } - @Test fun testCancelWithDelay() { // see https://github.com/Kotlin/kotlinx.coroutines/issues/586 diff --git a/kotlinx-coroutines-core/concurrent/test/exceptions/ConcurrentExceptionsStressTest.kt b/kotlinx-coroutines-core/concurrent/test/exceptions/ConcurrentExceptionsStressTest.kt new file mode 100644 index 0000000000..f0ea8d70d4 --- /dev/null +++ b/kotlinx-coroutines-core/concurrent/test/exceptions/ConcurrentExceptionsStressTest.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.exceptions + +import kotlinx.coroutines.* +import kotlin.test.* + +class ConcurrentExceptionsStressTest : TestBase() { + private val nWorkers = 4 + private val nRepeat = 1000 * stressTestMultiplier + + private val workers = Array(nWorkers) { index -> + newSingleThreadContext("JobExceptionsStressTest-$index") + } + + @AfterTest + fun tearDown() { + workers.forEach { + it.close() + } + } + + @Test + @Ignore // todo: this test is leaking memory on Kotlin/Native + fun testStress() = runTest { + repeat(nRepeat) { + testOnce() + } + } + + @Suppress("SuspendFunctionOnCoroutineScope") // workaround native inline fun stacktraces + private suspend fun CoroutineScope.testOnce() { + val deferred = async(NonCancellable) { + repeat(nWorkers) { index -> + // Always launch a coroutine even if parent job was already cancelled (atomic start) + launch(workers[index], start = CoroutineStart.ATOMIC) { + randomWait() + throw StressException(index) + } + } + } + deferred.join() + assertTrue(deferred.isCancelled) + val completionException = deferred.getCompletionExceptionOrNull() + val cause = completionException as? StressException + ?: unexpectedException("completion", completionException) + val suppressed = cause.suppressed + val indices = listOf(cause.index) + suppressed.mapIndexed { index, e -> + (e as? StressException)?.index ?: unexpectedException("suppressed $index", e) + } + repeat(nWorkers) { index -> + assertTrue(index in indices, "Exception $index is missing: $indices") + } + assertEquals(nWorkers, indices.size, "Duplicated exceptions in list: $indices") + } + + private fun unexpectedException(msg: String, e: Throwable?): Nothing { + e?.printStackTrace() + throw IllegalStateException("Unexpected $msg exception", e) + } + + private class StressException(val index: Int) : SuppressSupportingThrowable() +} + diff --git a/kotlinx-coroutines-core/concurrent/test/sync/MutexStressTest.kt b/kotlinx-coroutines-core/concurrent/test/sync/MutexStressTest.kt deleted file mode 100644 index 73b62aee2e..0000000000 --- a/kotlinx-coroutines-core/concurrent/test/sync/MutexStressTest.kt +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.sync - -import kotlinx.coroutines.* -import kotlinx.coroutines.exceptions.* -import kotlinx.coroutines.selects.* -import kotlin.test.* - -class MutexStressTest : TestBase() { - - private val n = (if (isNative) 1_000 else 10_000) * stressTestMultiplier - - @Test - fun testDefaultDispatcher() = runMtTest { testBody(Dispatchers.Default) } - - @Test - fun testSingleThreadContext() = runMtTest { - newSingleThreadContext("testSingleThreadContext").use { - testBody(it) - } - } - - @Test - fun testMultiThreadedContextWithSingleWorker() = runMtTest { - newFixedThreadPoolContext(1, "testMultiThreadedContextWithSingleWorker").use { - testBody(it) - } - } - - @Test - fun testMultiThreadedContext() = runMtTest { - newFixedThreadPoolContext(8, "testMultiThreadedContext").use { - testBody(it) - } - } - - @Suppress("SuspendFunctionOnCoroutineScope") - private suspend fun CoroutineScope.testBody(dispatcher: CoroutineDispatcher) { - val k = 100 - var shared = 0 - val mutex = Mutex() - val jobs = List(n) { - launch(dispatcher) { - repeat(k) { - mutex.lock() - shared++ - mutex.unlock() - } - } - } - jobs.forEach { it.join() } - assertEquals(n * k, shared) - } - - @Test - fun stressUnlockCancelRace() = runMtTest { - val n = 10_000 * stressTestMultiplier - val mutex = Mutex(true) // create a locked mutex - newSingleThreadContext("SemaphoreStressTest").use { pool -> - repeat(n) { - // Initially, we hold the lock and no one else can `lock`, - // otherwise it's a bug. - assertTrue(mutex.isLocked) - var job1EnteredCriticalSection = false - val job1 = launch(start = CoroutineStart.UNDISPATCHED) { - mutex.lock() - job1EnteredCriticalSection = true - mutex.unlock() - } - // check that `job1` didn't finish the call to `acquire()` - assertEquals(false, job1EnteredCriticalSection) - val job2 = launch(pool) { - mutex.unlock() - } - // Because `job2` executes in a separate thread, this - // cancellation races with the call to `unlock()`. - job1.cancelAndJoin() - job2.join() - assertFalse(mutex.isLocked) - mutex.lock() - } - } - } - - @Test - fun stressUnlockCancelRaceWithSelect() = runMtTest { - val n = 10_000 * stressTestMultiplier - val mutex = Mutex(true) // create a locked mutex - newSingleThreadContext("SemaphoreStressTest").use { pool -> - repeat(n) { - // Initially, we hold the lock and no one else can `lock`, - // otherwise it's a bug. - assertTrue(mutex.isLocked) - var job1EnteredCriticalSection = false - val job1 = launch(start = CoroutineStart.UNDISPATCHED) { - select { - mutex.onLock { - job1EnteredCriticalSection = true - mutex.unlock() - } - } - } - // check that `job1` didn't finish the call to `acquire()` - assertEquals(false, job1EnteredCriticalSection) - val job2 = launch(pool) { - mutex.unlock() - } - // Because `job2` executes in a separate thread, this - // cancellation races with the call to `unlock()`. - job1.cancelAndJoin() - job2.join() - assertFalse(mutex.isLocked) - mutex.lock() - } - } - } - - @Test - fun testShouldBeUnlockedOnCancellation() = runMtTest { - val mutex = Mutex() - val n = 1000 * stressTestMultiplier - repeat(n) { - val job = launch(Dispatchers.Default) { - mutex.lock() - mutex.unlock() - } - mutex.withLock { - job.cancel() - } - job.join() - assertFalse { mutex.isLocked } - } - } -} diff --git a/kotlinx-coroutines-core/js/src/Builders.kt b/kotlinx-coroutines-core/js/src/Builders.kt new file mode 100644 index 0000000000..37b589d007 --- /dev/null +++ b/kotlinx-coroutines-core/js/src/Builders.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.coroutines.intrinsics.* +import kotlin.coroutines.* +import kotlin.coroutines.intrinsics.* + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun startCoroutine( + start: CoroutineStart, + receiver: R, + completion: Continuation, + noinline onCancellation: ((cause: Throwable) -> Unit)?, + noinline block: suspend R.() -> T +) = + startCoroutineImpl(start, receiver, completion, onCancellation, block) + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun startAbstractCoroutine( + start: CoroutineStart, + receiver: R, + coroutine: AbstractCoroutine, + noinline block: suspend R.() -> T +) { + startCoroutineImpl(start, receiver, coroutine, null, block) +} + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun saveLazyCoroutine( + coroutine: AbstractCoroutine, + receiver: R, + noinline block: suspend R.() -> T +): Any = + block.createCoroutineUnintercepted(receiver, coroutine) + +@Suppress("NOTHING_TO_INLINE", "UNCHECKED_CAST") // Save an entry on call stack +internal actual inline fun startLazyCoroutine( + saved: Any, + coroutine: AbstractCoroutine, + receiver: R +) = + (saved as Continuation).startCoroutineCancellable(coroutine) diff --git a/kotlinx-coroutines-core/js/src/CoroutineContext.kt b/kotlinx-coroutines-core/js/src/CoroutineContext.kt index 8036c88a10..2e7c8a87cf 100644 --- a/kotlinx-coroutines-core/js/src/CoroutineContext.kt +++ b/kotlinx-coroutines-core/js/src/CoroutineContext.kt @@ -55,6 +55,6 @@ internal actual val CoroutineContext.coroutineName: String? get() = null // not internal actual class UndispatchedCoroutine actual constructor( context: CoroutineContext, uCont: Continuation -) : ScopeCoroutine(context, uCont) { +) : ScopeCoroutine(context, uCont, true) { override fun afterResume(state: Any?) = uCont.resumeWith(recoverResult(state, uCont)) } diff --git a/kotlinx-coroutines-core/js/src/EventLoop.kt b/kotlinx-coroutines-core/js/src/EventLoop.kt index 13c336969b..c4680bb3b3 100644 --- a/kotlinx-coroutines-core/js/src/EventLoop.kt +++ b/kotlinx-coroutines-core/js/src/EventLoop.kt @@ -8,6 +8,8 @@ import kotlin.coroutines.* internal actual fun createEventLoop(): EventLoop = UnconfinedEventLoop() +internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit) = block() + internal actual fun nanoTime(): Long = unsupported() internal class UnconfinedEventLoop : EventLoop() { @@ -25,5 +27,3 @@ internal actual object DefaultExecutor { private fun unsupported(): Nothing = throw UnsupportedOperationException("runBlocking event loop is not supported") - -internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit) = block() diff --git a/kotlinx-coroutines-core/js/src/Exceptions.kt b/kotlinx-coroutines-core/js/src/Exceptions.kt index da9979b603..c4be6ff264 100644 --- a/kotlinx-coroutines-core/js/src/Exceptions.kt +++ b/kotlinx-coroutines-core/js/src/Exceptions.kt @@ -20,8 +20,9 @@ public actual typealias CancellationException = kotlin.coroutines.cancellation.C internal actual class JobCancellationException public actual constructor( message: String, cause: Throwable?, - internal actual val job: Job + job: Job ) : CancellationException(message, cause) { + internal actual val job: Job? = job override fun toString(): String = "${super.toString()}; job=$job" override fun equals(other: Any?): Boolean = other === this || diff --git a/kotlinx-coroutines-core/js/src/channels/ArrayBufferState.kt b/kotlinx-coroutines-core/js/src/channels/ArrayBufferState.kt new file mode 100644 index 0000000000..0b985c886b --- /dev/null +++ b/kotlinx-coroutines-core/js/src/channels/ArrayBufferState.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +internal actual open class ArrayBufferState actual constructor(initialBufferSize: Int) { + protected var buffer: Array = arrayOfNulls(initialBufferSize) + + actual val bufferSize: Int get() = buffer.size + + actual fun getBufferAt(index: Int): Any? = + buffer[index] + + actual fun setBufferAt(index: Int, value: Any?) { + buffer[index] = value + } + + actual inline fun withLock(block: () -> T): T = block() +} diff --git a/kotlinx-coroutines-core/js/src/channels/ArrayChannelState.kt b/kotlinx-coroutines-core/js/src/channels/ArrayChannelState.kt new file mode 100644 index 0000000000..4d81aabf03 --- /dev/null +++ b/kotlinx-coroutines-core/js/src/channels/ArrayChannelState.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +import kotlin.math.* + +internal actual class ArrayChannelState actual constructor(initialBufferSize: Int) : ArrayBufferState(initialBufferSize) { + actual var head = 0 + actual var size = 0 + + actual fun ensureCapacity(currentSize: Int, capacity: Int) { + if (currentSize < buffer.size) return + val newSize = min(buffer.size * 2, capacity) + val newBuffer = arrayOfNulls(newSize) + for (i in 0 until currentSize) { + newBuffer[i] = buffer[(head + i) % buffer.size] + } + buffer = newBuffer + head = 0 + } +} diff --git a/kotlinx-coroutines-core/js/src/channels/ConflatedChannelState.kt b/kotlinx-coroutines-core/js/src/channels/ConflatedChannelState.kt new file mode 100644 index 0000000000..91c205aba6 --- /dev/null +++ b/kotlinx-coroutines-core/js/src/channels/ConflatedChannelState.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +internal actual class ConflatedChannelState { + actual var value: Any? = EMPTY + + actual inline fun withLock(block: () -> T): T = block() +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/js/src/internal/Concurrent.kt b/kotlinx-coroutines-core/js/src/internal/Concurrent.kt index 71f652271a..055998abe6 100644 --- a/kotlinx-coroutines-core/js/src/internal/Concurrent.kt +++ b/kotlinx-coroutines-core/js/src/internal/Concurrent.kt @@ -4,15 +4,6 @@ package kotlinx.coroutines.internal -internal actual typealias ReentrantLock = NoOpLock - -internal actual inline fun ReentrantLock.withLock(action: () -> T) = action() - -internal class NoOpLock { - fun tryLock() = true - fun unlock(): Unit {} -} - internal actual fun subscriberList(): SubscribersList = CopyOnWriteList() internal actual fun identitySet(expectedSize: Int): MutableSet = HashSet(expectedSize) diff --git a/kotlinx-coroutines-core/js/src/internal/LinkedList.kt b/kotlinx-coroutines-core/js/src/internal/LinkedList.kt index d8c07f4e19..48010913ee 100644 --- a/kotlinx-coroutines-core/js/src/internal/LinkedList.kt +++ b/kotlinx-coroutines-core/js/src/internal/LinkedList.kt @@ -146,7 +146,7 @@ public actual abstract class AbstractAtomicDesc : AtomicDesc() { } actual final override fun complete(op: AtomicOp<*>, failure: Any?) = onComplete() - protected actual open fun failure(affected: LockFreeLinkedListNode): Any? = null // Never fails by default + protected actual open fun failure(affected: LockFreeLinkedListNode?): Any? = null // Never fails by default // must fail on null for unlinked nodes on K/N protected actual open fun retry(affected: LockFreeLinkedListNode, next: Any): Boolean = false // Always succeeds protected actual abstract fun finishOnSuccess(affected: LockFreeLinkedListNode, next: LockFreeLinkedListNode) } diff --git a/kotlinx-coroutines-core/js/src/internal/ManualMemoryManagement.kt b/kotlinx-coroutines-core/js/src/internal/ManualMemoryManagement.kt new file mode 100644 index 0000000000..2df7a8e8ea --- /dev/null +++ b/kotlinx-coroutines-core/js/src/internal/ManualMemoryManagement.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.internal + +@Suppress("NOTHING_TO_INLINE") +internal actual inline fun disposeLockFreeLinkedList(list: () -> LockFreeLinkedListNode?) {} // only needed on Kotlin/Native + +@Suppress("NOTHING_TO_INLINE") +internal actual inline fun storeCyclicRef(block: () -> Unit) = block() diff --git a/kotlinx-coroutines-core/js/src/internal/Sharing.kt b/kotlinx-coroutines-core/js/src/internal/Sharing.kt new file mode 100644 index 0000000000..4c4c037be4 --- /dev/null +++ b/kotlinx-coroutines-core/js/src/internal/Sharing.kt @@ -0,0 +1,73 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.internal + +import kotlinx.coroutines.* +import kotlin.coroutines.* +import kotlin.coroutines.intrinsics.* + +@Suppress("ACTUAL_WITHOUT_EXPECT") // visibility different +internal actual typealias ShareableRefHolder = Any + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual fun ShareableRefHolder.disposeSharedRef() {} + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual fun T.asShareable(): DisposableHandle where T : DisposableHandle, T : ShareableRefHolder = this + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun CoroutineDispatcher.asShareable(): CoroutineDispatcher = this + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.asShareable() : Continuation = this + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.asLocal() : Continuation = this + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.asLocalOrNull() : Continuation? = this + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual fun Continuation.asLocalOrNullIfNotUsed() : Continuation? = this + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.useLocal() : Continuation = this + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun Continuation.shareableInterceptedResumeCancellableWith(result: Result) { + intercepted().resumeCancellableWith(result) +} + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun Continuation.shareableInterceptedResumeWith(result: Result) { + intercepted().resumeWith(result) +} + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun disposeContinuation(cont: () -> Continuation<*>) {} + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun CancellableContinuationImpl.shareableResume(delegate: Continuation, undispatched: Boolean) = + resume(delegate, undispatched) + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun (suspend (T) -> R).asShareable(): suspend (T) -> R = this + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun isReuseSupportedInPlatform() = true + +internal actual inline fun ArrayList.addOrUpdate(element: T, update: (ArrayList) -> Unit) { + add(element) +} + +internal actual inline fun ArrayList.addOrUpdate(index: Int, element: T, update: (ArrayList) -> Unit) { + add(index, element) +} + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Any.weakRef(): Any = this + +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Any?.unweakRef(): Any? = this diff --git a/kotlinx-coroutines-core/js/src/internal/Synchronized.kt b/kotlinx-coroutines-core/js/src/internal/Synchronized.kt index dcbb20217d..fd4ef3104d 100644 --- a/kotlinx-coroutines-core/js/src/internal/Synchronized.kt +++ b/kotlinx-coroutines-core/js/src/internal/Synchronized.kt @@ -18,3 +18,5 @@ public actual typealias SynchronizedObject = Any @InternalCoroutinesApi public actual inline fun synchronized(lock: SynchronizedObject, block: () -> T): T = block() + +internal actual val isNativeMt: Boolean = false diff --git a/kotlinx-coroutines-core/jvm/src/Builders.kt b/kotlinx-coroutines-core/jvm/src/Builders.kt index 8c4b62b19d..de8892d6b7 100644 --- a/kotlinx-coroutines-core/jvm/src/Builders.kt +++ b/kotlinx-coroutines-core/jvm/src/Builders.kt @@ -8,9 +8,11 @@ package kotlinx.coroutines +import kotlinx.coroutines.intrinsics.* import java.util.concurrent.locks.* import kotlin.contracts.* import kotlin.coroutines.* +import kotlin.coroutines.intrinsics.* /** * Runs a new coroutine and **blocks** the current thread _interruptibly_ until its completion. @@ -99,3 +101,39 @@ private class BlockingCoroutine( return state as T } } + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun startCoroutine( + start: CoroutineStart, + receiver: R, + completion: Continuation, + noinline onCancellation: ((cause: Throwable) -> Unit)?, + noinline block: suspend R.() -> T +) = + startCoroutineImpl(start, receiver, completion, onCancellation, block) + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun startAbstractCoroutine( + start: CoroutineStart, + receiver: R, + coroutine: AbstractCoroutine, + noinline block: suspend R.() -> T +) { + startCoroutineImpl(start, receiver, coroutine, null, block) +} + +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun saveLazyCoroutine( + coroutine: AbstractCoroutine, + receiver: R, + noinline block: suspend R.() -> T +): Any = + block.createCoroutineUnintercepted(receiver, coroutine) + +@Suppress("NOTHING_TO_INLINE", "UNCHECKED_CAST") // Save an entry on call stack +internal actual inline fun startLazyCoroutine( + saved: Any, + coroutine: AbstractCoroutine, + receiver: R +) = + (saved as Continuation).startCoroutineCancellable(coroutine) diff --git a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt index 7209bee803..86c314ab2d 100644 --- a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt +++ b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt @@ -5,9 +5,23 @@ package kotlinx.coroutines import kotlinx.coroutines.internal.* +import kotlinx.coroutines.scheduling.* +import kotlinx.coroutines.scheduling.DefaultScheduler import kotlin.coroutines.* import kotlin.coroutines.jvm.internal.CoroutineStackFrame +internal const val COROUTINES_SCHEDULER_PROPERTY_NAME = "kotlinx.coroutines.scheduler" + +internal val useCoroutinesScheduler = systemProp(COROUTINES_SCHEDULER_PROPERTY_NAME).let { value -> + when (value) { + null, "", "on" -> true + "off" -> false + else -> error("System property '$COROUTINES_SCHEDULER_PROPERTY_NAME' has unrecognized value '$value'") + } +} + +internal fun createDefaultDispatcher(): CoroutineDispatcher = DefaultScheduler + /** * Creates a context for a new coroutine. It installs [Dispatchers.Default] when no other dispatcher or * [ContinuationInterceptor] is specified and adds optional support for debugging facilities (when turned on) @@ -165,7 +179,7 @@ private object UndispatchedMarker: CoroutineContext.Element, CoroutineContext.Ke internal actual class UndispatchedCoroutineactual constructor ( context: CoroutineContext, uCont: Continuation -) : ScopeCoroutine(if (context[UndispatchedMarker] == null) context + UndispatchedMarker else context, uCont) { +) : ScopeCoroutine(if (context[UndispatchedMarker] == null) context + UndispatchedMarker else context, uCont, true) { /* * The state is thread-local because this coroutine can be used concurrently. diff --git a/kotlinx-coroutines-core/jvm/src/EventLoop.kt b/kotlinx-coroutines-core/jvm/src/EventLoop.kt index 1ee651aa41..4622f0574f 100644 --- a/kotlinx-coroutines-core/jvm/src/EventLoop.kt +++ b/kotlinx-coroutines-core/jvm/src/EventLoop.kt @@ -24,6 +24,8 @@ internal class BlockingEventLoop( internal actual fun createEventLoop(): EventLoop = BlockingEventLoop(Thread.currentThread()) +internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit) = block() + /** * Processes next event in the current thread's event loop. * @@ -46,5 +48,3 @@ internal actual fun createEventLoop(): EventLoop = BlockingEventLoop(Thread.curr @InternalCoroutinesApi public fun processNextEventInCurrentThread(): Long = ThreadLocalEventLoop.currentOrNull()?.processNextEvent() ?: Long.MAX_VALUE - -internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit) = block() diff --git a/kotlinx-coroutines-core/jvm/src/Exceptions.kt b/kotlinx-coroutines-core/jvm/src/Exceptions.kt index 007a0c98fa..5973579496 100644 --- a/kotlinx-coroutines-core/jvm/src/Exceptions.kt +++ b/kotlinx-coroutines-core/jvm/src/Exceptions.kt @@ -29,8 +29,9 @@ public actual fun CancellationException(message: String?, cause: Throwable?) : C internal actual class JobCancellationException public actual constructor( message: String, cause: Throwable?, - @JvmField internal actual val job: Job + job: Job ) : CancellationException(message), CopyableThrowable { + @JvmField internal actual val job: Job? = job init { if (cause != null) initCause(cause) @@ -52,7 +53,7 @@ internal actual class JobCancellationException public actual constructor( override fun createCopy(): JobCancellationException? { if (DEBUG) { - return JobCancellationException(message!!, this, job) + return JobCancellationException(message!!, this, job!!) } /* diff --git a/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt b/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt index dc0b7e29c5..b73ac88d81 100644 --- a/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt +++ b/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt @@ -5,10 +5,11 @@ package kotlinx.coroutines import java.util.concurrent.* -import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.* /** - * Creates a coroutine execution context using a single thread with built-in [yield] support. + * Creates a coroutine execution context using a single thread. + * * **NOTE: The resulting [ExecutorCoroutineDispatcher] owns native resources (its thread). * Resources are reclaimed by [ExecutorCoroutineDispatcher.close].** * @@ -59,7 +60,7 @@ public actual fun newSingleThreadContext(name: String): ExecutorCoroutineDispatc * @param name the base name of the created threads. */ @DelicateCoroutinesApi -public actual fun newFixedThreadPoolContext(nThreads: Int, name: String): ExecutorCoroutineDispatcher { +public fun newFixedThreadPoolContext(nThreads: Int, name: String): ExecutorCoroutineDispatcher { require(nThreads >= 1) { "Expected at least one thread, but $nThreads specified" } val threadNo = AtomicInteger() val executor = Executors.newScheduledThreadPool(nThreads) { runnable -> diff --git a/kotlinx-coroutines-core/jvm/src/channels/ArrayBufferState.kt b/kotlinx-coroutines-core/jvm/src/channels/ArrayBufferState.kt new file mode 100644 index 0000000000..eaefe40db9 --- /dev/null +++ b/kotlinx-coroutines-core/jvm/src/channels/ArrayBufferState.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +internal actual open class ArrayBufferState actual constructor(initialBufferSize: Int) { + protected var buffer: Array = arrayOfNulls(initialBufferSize) + + actual val bufferSize: Int get() = buffer.size + + actual fun getBufferAt(index: Int): Any? = + buffer[index] + + actual fun setBufferAt(index: Int, value: Any?) { + buffer[index] = value + } + + actual inline fun withLock(block: () -> T): T = + synchronized(this) { + block() + } +} diff --git a/kotlinx-coroutines-core/jvm/src/channels/ArrayChannelState.kt b/kotlinx-coroutines-core/jvm/src/channels/ArrayChannelState.kt new file mode 100644 index 0000000000..4d81aabf03 --- /dev/null +++ b/kotlinx-coroutines-core/jvm/src/channels/ArrayChannelState.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +import kotlin.math.* + +internal actual class ArrayChannelState actual constructor(initialBufferSize: Int) : ArrayBufferState(initialBufferSize) { + actual var head = 0 + actual var size = 0 + + actual fun ensureCapacity(currentSize: Int, capacity: Int) { + if (currentSize < buffer.size) return + val newSize = min(buffer.size * 2, capacity) + val newBuffer = arrayOfNulls(newSize) + for (i in 0 until currentSize) { + newBuffer[i] = buffer[(head + i) % buffer.size] + } + buffer = newBuffer + head = 0 + } +} diff --git a/kotlinx-coroutines-core/jvm/src/channels/ConflatedChannelState.kt b/kotlinx-coroutines-core/jvm/src/channels/ConflatedChannelState.kt new file mode 100644 index 0000000000..f5db45fbf4 --- /dev/null +++ b/kotlinx-coroutines-core/jvm/src/channels/ConflatedChannelState.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +internal actual class ConflatedChannelState { + actual var value: Any? = EMPTY + + actual inline fun withLock(block: () -> T): T = + synchronized(this) { + block() + } +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt b/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt index 050b974755..602d96732f 100644 --- a/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt +++ b/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt @@ -7,18 +7,10 @@ package kotlinx.coroutines.internal import java.lang.reflect.* import java.util.* import java.util.concurrent.* -import kotlin.concurrent.withLock as withLockJvm internal actual fun subscriberList(): SubscribersList = CopyOnWriteArrayList() -@Suppress("ACTUAL_WITHOUT_EXPECT") -internal actual typealias ReentrantLock = java.util.concurrent.locks.ReentrantLock - -internal actual inline fun ReentrantLock.withLock(action: () -> T) = this.withLockJvm(action) - -@Suppress("NOTHING_TO_INLINE") // So that R8 can completely remove ConcurrentKt class -internal actual inline fun identitySet(expectedSize: Int): MutableSet = - Collections.newSetFromMap(IdentityHashMap(expectedSize)) +internal actual fun identitySet(expectedSize: Int): MutableSet = Collections.newSetFromMap(IdentityHashMap(expectedSize)) private val REMOVE_FUTURE_ON_CANCEL: Method? = try { ScheduledThreadPoolExecutor::class.java.getMethod("setRemoveOnCancelPolicy", Boolean::class.java) diff --git a/kotlinx-coroutines-core/jvm/src/internal/ManualMemoryManagement.kt b/kotlinx-coroutines-core/jvm/src/internal/ManualMemoryManagement.kt new file mode 100644 index 0000000000..5eed134b7b --- /dev/null +++ b/kotlinx-coroutines-core/jvm/src/internal/ManualMemoryManagement.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") + +package kotlinx.coroutines.internal + +import kotlin.internal.* + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun disposeLockFreeLinkedList(list: () -> LockFreeLinkedListNode?) {} // only needed on Kotlin/Native + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") +internal actual inline fun storeCyclicRef(block: () -> Unit) = block() diff --git a/kotlinx-coroutines-core/jvm/src/internal/Sharing.kt b/kotlinx-coroutines-core/jvm/src/internal/Sharing.kt new file mode 100644 index 0000000000..4ae4c0d25d --- /dev/null +++ b/kotlinx-coroutines-core/jvm/src/internal/Sharing.kt @@ -0,0 +1,94 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") + +package kotlinx.coroutines.internal + +import kotlinx.coroutines.* +import kotlin.coroutines.* +import kotlin.coroutines.intrinsics.* +import kotlin.internal.* + +@Suppress("ACTUAL_WITHOUT_EXPECT") // visibility different +internal actual typealias ShareableRefHolder = Any + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual fun ShareableRefHolder.disposeSharedRef() {} + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual fun T.asShareable(): DisposableHandle where T : DisposableHandle, T : ShareableRefHolder = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun CoroutineDispatcher.asShareable(): CoroutineDispatcher = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.asShareable() : Continuation = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.asLocal() : Continuation = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.asLocalOrNull() : Continuation? = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual fun Continuation.asLocalOrNullIfNotUsed() : Continuation? = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.useLocal() : Continuation = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.shareableInterceptedResumeCancellableWith(result: Result) { + intercepted().resumeCancellableWith(result) +} + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Continuation.shareableInterceptedResumeWith(result: Result) { + intercepted().resumeWith(result) +} + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun disposeContinuation(cont: () -> Continuation<*>) {} + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun CancellableContinuationImpl.shareableResume(delegate: Continuation, undispatched: Boolean) = + resume(delegate, undispatched) + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun (suspend (T) -> R).asShareable(): suspend (T) -> R = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Save an entry on call stack +internal actual inline fun isReuseSupportedInPlatform() = true + +@InlineOnly +internal actual inline fun ArrayList.addOrUpdate(element: T, update: (ArrayList) -> Unit) { + add(element) +} + +@InlineOnly +internal actual inline fun ArrayList.addOrUpdate(index: Int, element: T, update: (ArrayList) -> Unit) { + add(index, element) +} + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Any.weakRef(): Any = this + +@InlineOnly +@Suppress("NOTHING_TO_INLINE") // Should be NOP +internal actual inline fun Any?.unweakRef(): Any? = this diff --git a/kotlinx-coroutines-core/jvm/src/internal/Synchronized.kt b/kotlinx-coroutines-core/jvm/src/internal/Synchronized.kt index 6284f3a099..dbc090187d 100644 --- a/kotlinx-coroutines-core/jvm/src/internal/Synchronized.kt +++ b/kotlinx-coroutines-core/jvm/src/internal/Synchronized.kt @@ -18,3 +18,5 @@ public actual typealias SynchronizedObject = Any @InternalCoroutinesApi public actual inline fun synchronized(lock: SynchronizedObject, block: () -> T): T = kotlin.synchronized(lock, block) + +internal actual val isNativeMt: Boolean = false diff --git a/kotlinx-coroutines-core/jvm/test/exceptions/Exceptions.kt b/kotlinx-coroutines-core/jvm/test/exceptions/Exceptions.kt index 4849f52071..b8a25ca681 100644 --- a/kotlinx-coroutines-core/jvm/test/exceptions/Exceptions.kt +++ b/kotlinx-coroutines-core/jvm/test/exceptions/Exceptions.kt @@ -1,5 +1,5 @@ /* - * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines.exceptions diff --git a/kotlinx-coroutines-core/native/src/Builders.kt b/kotlinx-coroutines-core/native/src/Builders.kt index f5b2222409..35621a43d0 100644 --- a/kotlinx-coroutines-core/native/src/Builders.kt +++ b/kotlinx-coroutines-core/native/src/Builders.kt @@ -5,8 +5,7 @@ @file:OptIn(ExperimentalContracts::class) package kotlinx.coroutines -import kotlinx.cinterop.* -import platform.posix.* +import kotlinx.coroutines.internal.* import kotlin.contracts.* import kotlin.coroutines.* import kotlin.native.concurrent.* @@ -43,7 +42,7 @@ public actual fun runBlocking(context: CoroutineContext, block: suspend Coro if (contextInterceptor == null) { // create or use private event loop if no dispatcher is specified eventLoop = ThreadLocalEventLoop.eventLoop - newContext = GlobalScope.newCoroutineContext(context + eventLoop) + newContext = GlobalScope.newCoroutineContext(context + eventLoop.asShareable()) } else { // See if context's interceptor is an event loop that we shall use (to support TestContext) // or take an existing thread-local event loop if present to avoid blocking it (but don't create one) @@ -51,49 +50,120 @@ public actual fun runBlocking(context: CoroutineContext, block: suspend Coro ?: ThreadLocalEventLoop.currentOrNull() newContext = GlobalScope.newCoroutineContext(context) } - val coroutine = BlockingCoroutine(newContext, eventLoop) + val coroutine = BlockingCoroutine(newContext) coroutine.start(CoroutineStart.DEFAULT, coroutine, block) - return coroutine.joinBlocking() + return coroutine.joinBlocking(eventLoop) } private class BlockingCoroutine( - parentContext: CoroutineContext, - private val eventLoop: EventLoop? + parentContext: CoroutineContext ) : AbstractCoroutine(parentContext, true, true) { private val joinWorker = Worker.current override val isScopedCoroutine: Boolean get() = true + private val worker = Worker.current override fun afterCompletion(state: Any?) { - // wake up blocked thread - if (joinWorker != Worker.current) { - // Unpark waiting worker - joinWorker.executeAfter(0L, {}) // send an empty task to unpark the waiting event loop - } + // wake up blocked worker + if (Worker.current != worker) + worker.execute(TransferMode.SAFE, {}) {} // send an empty task } @Suppress("UNCHECKED_CAST") - fun joinBlocking(): T { - try { - eventLoop?.incrementUseCount() - while (true) { - var parkNanos: Long - // Workaround for bug in BE optimizer that cannot eliminate boxing here - if (eventLoop != null) { - parkNanos = eventLoop.processNextEvent() - } else { - parkNanos = Long.MAX_VALUE - } - // note: processNextEvent may lose unpark flag, so check if completed before parking - if (isCompleted) break - joinWorker.park(parkNanos / 1000L, true) - } - } finally { // paranoia - eventLoop?.decrementUseCount() - } + fun joinBlocking(eventLoop: EventLoop?): T { + runEventLoop(eventLoop) { isCompleted } // now return result val state = state.unboxState() (state as? CompletedExceptionally)?.let { throw it.cause } return state as T } } + +internal fun runEventLoop(eventLoop: EventLoop?, isCompleted: () -> Boolean) { + try { + eventLoop?.incrementUseCount() + val thread = currentThread() + while (!isCompleted()) { + val parkNanos = eventLoop?.processNextEvent() ?: Long.MAX_VALUE + if (isCompleted()) break + thread.parkNanos(parkNanos) + } + } finally { // paranoia + eventLoop?.decrementUseCount() + } +} + +// --------------- Kotlin/Native specialization hooks --------------- + +// just start +internal actual fun startCoroutine( + start: CoroutineStart, + receiver: R, + completion: Continuation, + onCancellation: ((cause: Throwable) -> Unit)?, + block: suspend R.() -> T +) = + startCoroutine(start, receiver, completion, onCancellation, block) {} + +// initParentJob + startCoroutine +internal actual fun startAbstractCoroutine( + start: CoroutineStart, + receiver: R, + coroutine: AbstractCoroutine, + + block: suspend R.() -> T +) { + // See https://github.com/Kotlin/kotlinx.coroutines/issues/2064 + // We shall do initParentJob only after freezing the block + startCoroutine(start, receiver, coroutine, null, block) { + coroutine.initParentJob(coroutine.parentContext[Job]) + } +} + +private fun startCoroutine( + start: CoroutineStart, + receiver: R, + completion: Continuation, + onCancellation: ((cause: Throwable) -> Unit)?, + block: suspend R.() -> T, + initParentJobIfNeeded: () -> Unit +) { + val curThread = currentThread() + val newThread = completion.context[ContinuationInterceptor].thread() + if (newThread != curThread) { + check(start != CoroutineStart.UNDISPATCHED) { + "Cannot start an undispatched coroutine in another thread $newThread from current $curThread" + } + block.freeze() // freeze the block, safely get FreezingException if it cannot be frozen + initParentJobIfNeeded() // only initParentJob here if needed + if (start != CoroutineStart.LAZY) { + newThread.execute { + startCoroutineImpl(start, receiver, completion, onCancellation, block) + } + } + return + } + initParentJobIfNeeded() + startCoroutineImpl(start, receiver, completion, onCancellation, block) +} + +private fun ContinuationInterceptor?.thread(): Thread = when (this) { + null -> Dispatchers.Default.thread() + is ThreadBoundInterceptor -> thread + else -> currentThread() // fallback +} + +internal actual fun saveLazyCoroutine( + coroutine: AbstractCoroutine, + receiver: R, + block: suspend R.() -> T +): Any = + block + +@Suppress("NOTHING_TO_INLINE", "UNCHECKED_CAST") // Save an entry on call stack +internal actual fun startLazyCoroutine( + saved: Any, + coroutine: AbstractCoroutine, + receiver: R +) = + startCoroutine(CoroutineStart.DEFAULT, receiver, coroutine, null, saved as suspend R.() -> T) diff --git a/kotlinx-coroutines-core/native/src/CloseableCoroutineDispatcher.kt b/kotlinx-coroutines-core/native/src/CloseableCoroutineDispatcher.kt deleted file mode 100644 index 0e239a42f7..0000000000 --- a/kotlinx-coroutines-core/native/src/CloseableCoroutineDispatcher.kt +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines - -public actual abstract class CloseableCoroutineDispatcher actual constructor() : CoroutineDispatcher() { - public actual abstract fun close() -} diff --git a/kotlinx-coroutines-core/native/src/CoroutineContext.kt b/kotlinx-coroutines-core/native/src/CoroutineContext.kt index 6e2dac1a29..a3db804473 100644 --- a/kotlinx-coroutines-core/native/src/CoroutineContext.kt +++ b/kotlinx-coroutines-core/native/src/CoroutineContext.kt @@ -8,45 +8,30 @@ import kotlinx.coroutines.internal.* import kotlin.coroutines.* import kotlin.native.concurrent.* -internal actual object DefaultExecutor : CoroutineDispatcher(), Delay { - - private val delegate = WorkerDispatcher(name = "DefaultExecutor") - - override fun dispatch(context: CoroutineContext, block: Runnable) { - checkState() - delegate.dispatch(context, block) - } - - override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) { - checkState() - delegate.scheduleResumeAfterDelay(timeMillis, continuation) - } +private fun takeEventLoop(): EventLoopImpl = + ThreadLocalEventLoop.currentOrNull() as? EventLoopImpl ?: + error("There is no event loop. Use runBlocking { ... } to start one.") - override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle { - checkState() - return delegate.invokeOnTimeout(timeMillis, block, context) - } - - actual fun enqueue(task: Runnable): Unit { - checkState() - delegate.dispatch(EmptyCoroutineContext, task) - } - - private fun checkState() { - if (multithreadingSupported) return - error("DefaultExecutor should never be invoked in K/N with disabled new memory model. The top-most 'runBlocking' event loop has been shutdown") - } +internal actual object DefaultExecutor : CoroutineDispatcher(), Delay { + override fun dispatch(context: CoroutineContext, block: Runnable) = + takeEventLoop().dispatch(context, block) + override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) = + takeEventLoop().scheduleResumeAfterDelay(timeMillis, continuation) + override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle = + takeEventLoop().invokeOnTimeout(timeMillis, block, context) + + actual fun enqueue(task: Runnable): Unit = loopWasShutDown() } -internal expect fun createDefaultDispatcher(): CoroutineDispatcher +internal fun loopWasShutDown(): Nothing = error("Cannot execute task because event loop was shut down") @SharedImmutable -internal actual val DefaultDelay: Delay = if (multithreadingSupported) DefaultExecutor else OldDefaultExecutor +internal actual val DefaultDelay: Delay = DefaultExecutor public actual fun CoroutineScope.newCoroutineContext(context: CoroutineContext): CoroutineContext { val combined = coroutineContext + context - return if (combined !== DefaultDelay && combined[ContinuationInterceptor] == null) - combined + (DefaultDelay as CoroutineContext.Element) else combined + return if (combined !== Dispatchers.Default && combined[ContinuationInterceptor] == null) + combined + Dispatchers.Default else combined } public actual fun CoroutineContext.newCoroutineContext(addedContext: CoroutineContext): CoroutineContext { @@ -62,6 +47,6 @@ internal actual val CoroutineContext.coroutineName: String? get() = null // not internal actual class UndispatchedCoroutine actual constructor( context: CoroutineContext, uCont: Continuation -) : ScopeCoroutine(context, uCont) { +) : ScopeCoroutine(context, uCont, true) { override fun afterResume(state: Any?) = uCont.resumeWith(recoverResult(state, uCont)) } diff --git a/kotlinx-coroutines-core/native/src/Dispatchers.kt b/kotlinx-coroutines-core/native/src/Dispatchers.kt deleted file mode 100644 index 6c51a03463..0000000000 --- a/kotlinx-coroutines-core/native/src/Dispatchers.kt +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines - -import kotlinx.coroutines.internal.multithreadingSupported -import kotlin.coroutines.* - -public actual object Dispatchers { - public actual val Default: CoroutineDispatcher = createDefaultDispatcherBasedOnMm() - public actual val Main: MainCoroutineDispatcher - get() = injectedMainDispatcher ?: mainDispatcher - public actual val Unconfined: CoroutineDispatcher get() = kotlinx.coroutines.Unconfined // Avoid freezing - - private val mainDispatcher = createMainDispatcher(Default) - - private var injectedMainDispatcher: MainCoroutineDispatcher? = null - - @PublishedApi - internal fun injectMain(dispatcher: MainCoroutineDispatcher) { - if (!multithreadingSupported) { - throw IllegalStateException("Dispatchers.setMain is not supported in Kotlin/Native when new memory model is disabled") - } - injectedMainDispatcher = dispatcher - } - - @PublishedApi - internal fun resetInjectedMain() { - injectedMainDispatcher = null - } -} - -internal expect fun createMainDispatcher(default: CoroutineDispatcher): MainCoroutineDispatcher - -private fun createDefaultDispatcherBasedOnMm(): CoroutineDispatcher { - return if (multithreadingSupported) createDefaultDispatcher() - else OldDefaultExecutor -} - -private fun takeEventLoop(): EventLoopImpl = - ThreadLocalEventLoop.currentOrNull() as? EventLoopImpl ?: - error("There is no event loop. Use runBlocking { ... } to start one.") - -internal object OldDefaultExecutor : CoroutineDispatcher(), Delay { - override fun dispatch(context: CoroutineContext, block: Runnable) = - takeEventLoop().dispatch(context, block) - override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) = - takeEventLoop().scheduleResumeAfterDelay(timeMillis, continuation) - override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle = - takeEventLoop().invokeOnTimeout(timeMillis, block, context) -} - -internal class OldMainDispatcher(private val delegate: CoroutineDispatcher) : MainCoroutineDispatcher() { - override val immediate: MainCoroutineDispatcher - get() = throw UnsupportedOperationException("Immediate dispatching is not supported on Native") - override fun dispatch(context: CoroutineContext, block: Runnable) = delegate.dispatch(context, block) - override fun isDispatchNeeded(context: CoroutineContext): Boolean = delegate.isDispatchNeeded(context) - override fun dispatchYield(context: CoroutineContext, block: Runnable) = delegate.dispatchYield(context, block) - override fun toString(): String = toStringInternalImpl() ?: delegate.toString() -} diff --git a/kotlinx-coroutines-core/native/src/Dispatchers.native.kt b/kotlinx-coroutines-core/native/src/Dispatchers.native.kt new file mode 100644 index 0000000000..40277aefb9 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/Dispatchers.native.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.atomicfu.* +import kotlinx.atomicfu.locks.* +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.multithreadingSupported +import kotlin.coroutines.* + +public actual object Dispatchers { + public actual val Default: CoroutineDispatcher get() = DefaultDispatcher + public actual val Main: MainCoroutineDispatcher = createMainDispatcher(Default) + public actual val Unconfined: CoroutineDispatcher get() = kotlinx.coroutines.Unconfined // Avoid freezing + + private var injectedMainDispatcher: MainCoroutineDispatcher? = null + + @PublishedApi + internal fun injectMain(dispatcher: MainCoroutineDispatcher) { + if (!multithreadingSupported) { + throw IllegalStateException("Dispatchers.setMain is not supported in Kotlin/Native when new memory model is disabled") + } + injectedMainDispatcher = dispatcher + } + + @PublishedApi + internal fun resetInjectedMain() { + injectedMainDispatcher = null + } +} + +internal expect fun createMainDispatcher(default: CoroutineDispatcher): MainCoroutineDispatcher + +// Create DefaultDispatcher thread only when explicitly requested +internal object DefaultDispatcher : CoroutineDispatcher(), Delay, ThreadBoundInterceptor { + private val lock = SynchronizedObject() + private val _delegate = atomic(null) +// private val delegate by lazy { newSingleThreadContext("DefaultDispatcher") } + private val delegate: CloseableCoroutineDispatcher + get() = _delegate.value ?: getOrCreateDefaultDispatcher() + + private fun getOrCreateDefaultDispatcher() = lock.withLock { + _delegate.value ?: newSingleThreadContext("DefaultDispatcher").also { _delegate.value = it } + } + + override val thread: Thread + get() = delegate.thread + override fun dispatch(context: CoroutineContext, block: Runnable) = + delegate.dispatch(context, block) + override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) = + (delegate as Delay).scheduleResumeAfterDelay(timeMillis, continuation) + override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle = + (delegate as Delay).invokeOnTimeout(timeMillis, block, context) + override fun toString(): String = + delegate.toString() + + // only for tests + internal fun shutdown() { + _delegate.getAndSet(null)?.close() + } +} diff --git a/kotlinx-coroutines-core/native/src/EventLoop.kt b/kotlinx-coroutines-core/native/src/EventLoop.kt index f4e5b8c9c4..23a72d1a6a 100644 --- a/kotlinx-coroutines-core/native/src/EventLoop.kt +++ b/kotlinx-coroutines-core/native/src/EventLoop.kt @@ -5,36 +5,68 @@ package kotlinx.coroutines import kotlinx.cinterop.* -import kotlinx.coroutines.internal.* -import kotlinx.coroutines.internal.multithreadingSupported -import platform.posix.* import kotlin.coroutines.* import kotlin.native.concurrent.* import kotlin.system.* -internal actual abstract class EventLoopImplPlatform : EventLoop() { - - private val current = Worker.current - +internal actual abstract class EventLoopImplPlatform: EventLoop() { protected actual fun unpark() { - current.executeAfter(0L, {})// send an empty task to unpark the waiting event loop + /* + * Does nothing, because we only work with EventLoop in Kotlin/Native from a single thread where + * it was created. All tasks that come from other threads are passed into the owner thread via + * Worker.execute and its queueing mechanics. + */ } - protected actual fun reschedule(now: Long, delayedTask: EventLoopImplBase.DelayedTask) { - if (multithreadingSupported) { - DefaultExecutor.invokeOnTimeout(now, delayedTask, EmptyCoroutineContext) - } else { - error("Cannot execute task because event loop was shut down") - } - } + protected actual fun reschedule(now: Long, delayedTask: EventLoopImplBase.DelayedTask): Unit = + loopWasShutDown() } internal class EventLoopImpl: EventLoopImplBase() { + init { ensureNeverFrozen() } + + val shareable = ShareableEventLoop(StableRef.create(this), Worker.current) + + override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle = + scheduleInvokeOnTimeout(timeMillis, block) + + override fun shutdown() { + super.shutdown() + shareable.ref.dispose() + } +} + +internal class ShareableEventLoop( + val ref: StableRef, + private val worker: Worker +) : CoroutineDispatcher(), Delay, ThreadBoundInterceptor { + override val thread: Thread = WorkerThread(worker) + + init { freeze() } + + override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) { + checkCurrentThread() + ref.get().scheduleResumeAfterDelay(timeMillis, continuation) + } + override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle { - if (!multithreadingSupported) { - return scheduleInvokeOnTimeout(timeMillis, block) - } - return DefaultDelay.invokeOnTimeout(timeMillis, block, context) + checkCurrentThread() + return ref.get().invokeOnTimeout(timeMillis, block, context) + } + + override fun dispatch(context: CoroutineContext, block: Runnable) { + checkCurrentThread() + ref.get().dispatch(context, block) + } + + override fun interceptContinuation(continuation: Continuation): Continuation { + checkCurrentThread() + return ref.get().interceptContinuation(continuation) + } + + override fun releaseInterceptedContinuation(continuation: Continuation<*>) { + checkCurrentThread() + ref.get().releaseInterceptedContinuation(continuation) } } diff --git a/kotlinx-coroutines-core/native/src/Exceptions.kt b/kotlinx-coroutines-core/native/src/Exceptions.kt index 1a923c40ff..62966fdd0c 100644 --- a/kotlinx-coroutines-core/native/src/Exceptions.kt +++ b/kotlinx-coroutines-core/native/src/Exceptions.kt @@ -6,6 +6,8 @@ package kotlinx.coroutines import kotlinx.coroutines.internal.* import kotlinx.coroutines.internal.SuppressSupportingThrowableImpl +import kotlinx.atomicfu.* +import kotlin.native.ref.* /** * Thrown by cancellable suspending functions if the [Job] of the coroutine is cancelled while it is suspending. @@ -23,8 +25,12 @@ public actual typealias CancellationException = kotlin.coroutines.cancellation.C internal actual class JobCancellationException public actual constructor( message: String, cause: Throwable?, - internal actual val job: Job + job: Job ) : CancellationException(message, cause) { + private val ref = WeakReference(job) + internal actual val job: Job? + get() = ref.get() + override fun toString(): String = "${super.toString()}; job=$job" override fun equals(other: Any?): Boolean = other === this || @@ -33,8 +39,7 @@ internal actual class JobCancellationException public actual constructor( (message!!.hashCode() * 31 + job.hashCode()) * 31 + (cause?.hashCode() ?: 0) } -@Suppress("NOTHING_TO_INLINE") -internal actual inline fun Throwable.addSuppressedThrowable(other: Throwable) { +internal actual fun Throwable.addSuppressedThrowable(other: Throwable) { if (this is SuppressSupportingThrowableImpl) addSuppressed(other) } diff --git a/kotlinx-coroutines-core/native/src/MultithreadedDispatchers.kt b/kotlinx-coroutines-core/native/src/MultithreadedDispatchers.kt deleted file mode 100644 index d991fc169e..0000000000 --- a/kotlinx-coroutines-core/native/src/MultithreadedDispatchers.kt +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines - -import kotlinx.coroutines.channels.* -import kotlinx.coroutines.internal.* -import kotlin.coroutines.* -import kotlin.native.concurrent.* - -@ExperimentalCoroutinesApi -public actual fun newSingleThreadContext(name: String): CloseableCoroutineDispatcher { - if (!multithreadingSupported) throw IllegalStateException("This API is only supported for experimental K/N memory model") - return WorkerDispatcher(name) -} - -public actual fun newFixedThreadPoolContext(nThreads: Int, name: String): CloseableCoroutineDispatcher { - if (!multithreadingSupported) throw IllegalStateException("This API is only supported for experimental K/N memory model") - require(nThreads >= 1) { "Expected at least one thread, but got: $nThreads"} - return MultiWorkerDispatcher(name, nThreads) -} - -internal class WorkerDispatcher(name: String) : CloseableCoroutineDispatcher(), Delay { - private val worker = Worker.start(name = name) - - override fun dispatch(context: CoroutineContext, block: Runnable) { - worker.executeAfter(0L) { block.run() } - } - - override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) { - worker.executeAfter(timeMillis.toMicrosSafe()) { - with(continuation) { resumeUndispatched(Unit) } - } - } - - override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle { - // No API to cancel on timeout - worker.executeAfter(timeMillis.toMicrosSafe()) { block.run() } - return NonDisposableHandle - } - - override fun close() { - worker.requestTermination().result // Note: calling "result" blocks - } - - private fun Long.toMicrosSafe(): Long { - val result = this * 1000 - return if (result > this) result else Long.MAX_VALUE - } -} - -private class MultiWorkerDispatcher(name: String, workersCount: Int) : CloseableCoroutineDispatcher() { - private val tasksQueue = Channel(Channel.UNLIMITED) - private val workers = Array(workersCount) { Worker.start(name = "$name-$it") } - - init { - workers.forEach { w -> w.executeAfter(0L) { workerRunLoop() } } - } - - private fun workerRunLoop() = runBlocking { - for (task in tasksQueue) { - // TODO error handling - task.run() - } - } - - override fun dispatch(context: CoroutineContext, block: Runnable) { - // TODO handle rejections - tasksQueue.trySend(block) - } - - override fun close() { - tasksQueue.close() - workers.forEach { it.requestTermination().result } - } -} diff --git a/kotlinx-coroutines-core/native/src/Thread.native.kt b/kotlinx-coroutines-core/native/src/Thread.native.kt new file mode 100644 index 0000000000..5b13688e11 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/Thread.native.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.native.concurrent.* + +internal abstract class Thread { + abstract fun execute(block: () -> Unit) + abstract fun parkNanos(timeout: Long) +} + +@ThreadLocal +private val currentThread: Thread = initCurrentThread() + +internal fun currentThread(): Thread = currentThread + +internal expect fun initCurrentThread(): Thread + +internal expect fun Worker.toThread(): Thread + +internal fun Worker.execute(block: () -> Unit) { + block.freeze() + executeAfter(0, block) +} + +internal open class WorkerThread(val worker: Worker = Worker.current) : Thread() { + override fun execute(block: () -> Unit) = worker.execute(block) + + override fun parkNanos(timeout: Long) { + // Note: worker is parked in microseconds + worker.park(timeout / 1000L, process = true) + } + + override fun equals(other: Any?): Boolean = other is WorkerThread && other.worker == worker + override fun hashCode(): Int = worker.hashCode() + override fun toString(): String = worker.name +} + +internal interface ThreadBoundInterceptor { + val thread: Thread +} + +internal fun ThreadBoundInterceptor.checkCurrentThread() { + val current = currentThread() + check(current == thread) { "This dispatcher can be used only from a single thread $thread, but now in $current" } +} diff --git a/kotlinx-coroutines-core/native/src/Workers.kt b/kotlinx-coroutines-core/native/src/Workers.kt new file mode 100644 index 0000000000..a83556c410 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/Workers.kt @@ -0,0 +1,84 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.atomicfu.* +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.multithreadingSupported +import kotlin.coroutines.* +import kotlin.native.concurrent.* +import kotlin.native.concurrent.freeze as stdlibFreeze + +/** + * Creates a coroutine execution context using a single thread. + */ +@ExperimentalCoroutinesApi +public actual fun newSingleThreadContext(name: String): CloseableCoroutineDispatcher = + WorkerCoroutineDispatcherImpl(name).apply { start() } + +/** + * A coroutine dispatcher that is confined to a single thread. + */ +@ExperimentalCoroutinesApi +@Suppress("ACTUAL_WITHOUT_EXPECT") +public actual abstract class CloseableCoroutineDispatcher actual constructor() : CoroutineDispatcher() { + /** + * A reference to this dispatcher's worker. + */ + @ExperimentalCoroutinesApi + public abstract val worker: Worker + + internal abstract val thread: Thread + + /** + * Closes this coroutine dispatcher and shuts down its thread. + */ + @ExperimentalCoroutinesApi + public actual abstract fun close() +} + +private class WorkerCoroutineDispatcherImpl(name: String) : CloseableCoroutineDispatcher(), ThreadBoundInterceptor, Delay { + override val worker = Worker.start(name = name) + override val thread = WorkerThread(worker) + private val isClosed = atomic(false) + + init { freeze() } + + fun start() { + worker.execute { + workerMain { + runEventLoop(ThreadLocalEventLoop.eventLoop) { isClosed.value } + } + } + } + + override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) { + checkCurrentThread() + (ThreadLocalEventLoop.eventLoop as Delay).scheduleResumeAfterDelay(timeMillis, continuation) + } + + override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle { + checkCurrentThread() + return (ThreadLocalEventLoop.eventLoop as Delay).invokeOnTimeout(timeMillis, block, context) + } + + override fun dispatch(context: CoroutineContext, block: Runnable) { + checkCurrentThread() + ThreadLocalEventLoop.eventLoop.dispatch(context, block) + } + + override fun close() { + isClosed.value = true + worker.requestTermination().result // Note: calling "result" blocks + } +} + +internal fun T.freeze(): T { + return if (multithreadingSupported) { + this + } else { + stdlibFreeze() + } +} diff --git a/kotlinx-coroutines-core/native/src/channels/ArrayBufferState.kt b/kotlinx-coroutines-core/native/src/channels/ArrayBufferState.kt new file mode 100644 index 0000000000..9f09e5a446 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/channels/ArrayBufferState.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +import kotlinx.atomicfu.* +import kotlinx.atomicfu.locks.* + +internal actual open class ArrayBufferState actual constructor(initialBufferSize: Int) : SynchronizedObject() { + protected val _buffer = atomic(atomicArrayOfNulls(initialBufferSize)) + protected val _bufferSize = atomic(initialBufferSize) + + actual val bufferSize: Int + get() = _bufferSize.value + + actual fun getBufferAt(index: Int): Any? = + _buffer.value[index].value + + actual fun setBufferAt(index: Int, value: Any?) { + _buffer.value[index].value = value + } + + actual inline fun withLock(block: () -> T): T = + synchronized(this) { + block() + } +} diff --git a/kotlinx-coroutines-core/native/src/channels/ArrayChannelState.kt b/kotlinx-coroutines-core/native/src/channels/ArrayChannelState.kt new file mode 100644 index 0000000000..fac45165a6 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/channels/ArrayChannelState.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +import kotlinx.atomicfu.* +import kotlin.math.* + +internal actual class ArrayChannelState actual constructor(initialBufferSize: Int) : ArrayBufferState(initialBufferSize) { + private val _head = atomic(0) + private val _size = atomic(0) + + actual var head: Int + get() = _head.value + set(value) { _head.value = value } + + actual var size: Int + get() = _size.value + set(value) { _size.value = value } + + actual fun ensureCapacity(currentSize: Int, capacity: Int) { + if (currentSize < bufferSize) return + val newSize = min(bufferSize * 2, capacity) + val newBuffer = atomicArrayOfNulls(newSize) + for (i in 0 until currentSize) { + newBuffer[i].value = _buffer.value[(head + i) % bufferSize].value + } + _buffer.value = newBuffer + _bufferSize.value = newSize + head = 0 + } +} diff --git a/kotlinx-coroutines-core/native/src/channels/ConflatedChannelState.kt b/kotlinx-coroutines-core/native/src/channels/ConflatedChannelState.kt new file mode 100644 index 0000000000..acc144e9e4 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/channels/ConflatedChannelState.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +import kotlinx.atomicfu.* +import kotlinx.atomicfu.locks.* + +internal actual class ConflatedChannelState : SynchronizedObject() { + private val _value = atomic(EMPTY) + + actual var value: Any? + get() = _value.value + set(value) { _value.value = value } + + actual inline fun withLock(block: () -> T): T = + synchronized(this) { + block() + } +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/native/src/internal/Concurrent.kt b/kotlinx-coroutines-core/native/src/internal/Concurrent.kt index f6e18dd5fc..6df4defaee 100644 --- a/kotlinx-coroutines-core/native/src/internal/Concurrent.kt +++ b/kotlinx-coroutines-core/native/src/internal/Concurrent.kt @@ -8,16 +8,10 @@ import kotlinx.atomicfu.* import kotlin.native.concurrent.* import kotlinx.atomicfu.locks.withLock as withLock2 -@Suppress("ACTUAL_WITHOUT_EXPECT") -internal actual typealias ReentrantLock = kotlinx.atomicfu.locks.SynchronizedObject - -internal actual inline fun ReentrantLock.withLock(action: () -> T): T = this.withLock2(action) - internal actual fun subscriberList(): MutableList = CopyOnWriteList() internal actual fun identitySet(expectedSize: Int): MutableSet = HashSet() - // "Suppress-supporting throwable" is currently used for tests only internal open class SuppressSupportingThrowableImpl : Throwable() { private val _suppressed = atomic>(emptyArray()) diff --git a/kotlinx-coroutines-core/native/src/internal/CopyOnWriteList.kt b/kotlinx-coroutines-core/native/src/internal/CopyOnWriteList.kt index 2896c2eac5..797cbdfdd4 100644 --- a/kotlinx-coroutines-core/native/src/internal/CopyOnWriteList.kt +++ b/kotlinx-coroutines-core/native/src/internal/CopyOnWriteList.kt @@ -7,9 +7,9 @@ package kotlinx.coroutines.internal import kotlinx.atomicfu.* @Suppress("UNCHECKED_CAST") -internal class CopyOnWriteList : AbstractMutableList() { +internal class CopyOnWriteList() : AbstractMutableList() { - private val _array = atomic(arrayOfNulls(0)) + private val _array = atomic>(arrayOfNulls(0)) private var array: Array get() = _array.value set(value) { _array.value = value } @@ -59,7 +59,7 @@ internal class CopyOnWriteList : AbstractMutableList() { override fun isEmpty(): Boolean = size == 0 override fun set(index: Int, element: E): E = throw UnsupportedOperationException("Operation is not supported") override fun get(index: Int): E = array[rangeCheck(index)] as E - + private class IteratorImpl(private val array: Array) : MutableIterator { private var current = 0 diff --git a/kotlinx-coroutines-core/native/src/internal/ManualMemoryManagement.kt b/kotlinx-coroutines-core/native/src/internal/ManualMemoryManagement.kt new file mode 100644 index 0000000000..454925b405 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/internal/ManualMemoryManagement.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.internal + +@Suppress("NOTHING_TO_INLINE") +internal actual inline fun disposeLockFreeLinkedList(list: () -> LockFreeLinkedListNode?) { + // only needed on Kotlin/Native + val head = list() ?: return + var cur = head + do { + val next = cur.nextNode // returns cur when already unlinked last node + val last = next === head || next === cur + cur.unlinkRefs(last) + cur = next + } while (!last) +} + +@Suppress("NOTHING_TO_INLINE") +internal actual inline fun storeCyclicRef(block: () -> Unit) {} // nop on native diff --git a/kotlinx-coroutines-core/native/src/internal/Sharing.kt b/kotlinx-coroutines-core/native/src/internal/Sharing.kt new file mode 100644 index 0000000000..e91218d371 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/internal/Sharing.kt @@ -0,0 +1,218 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.internal + +import kotlinx.atomicfu.* +import kotlinx.coroutines.* +import kotlin.coroutines.* +import kotlin.coroutines.intrinsics.* +import kotlin.native.concurrent.* +import kotlin.native.ref.* + +internal actual open class ShareableRefHolder { + internal var shareable: ShareableObject<*>? = null // cached result of asShareable call +} + +internal actual fun ShareableRefHolder.disposeSharedRef() { + shareable?.disposeRef() +} + +internal actual fun T.asShareable(): DisposableHandle where T : DisposableHandle, T : ShareableRefHolder { + shareable?.let { return it as DisposableHandle } + return ShareableDisposableHandle(this).also { shareable = it } +} + +internal actual fun CoroutineDispatcher.asShareable(): CoroutineDispatcher = when (this) { + is EventLoopImpl -> shareable + else -> this +} + +internal actual fun Continuation.asShareable() : Continuation = when (this) { + is ShareableContinuation -> this + else -> ShareableContinuation(this) +} + +internal actual fun Continuation.asLocal() : Continuation = when (this) { + is ShareableContinuation -> localRef() + else -> this +} + +internal actual fun Continuation.asLocalOrNull() : Continuation? = when (this) { + is ShareableContinuation -> localRefOrNull() + else -> this +} + +internal actual fun Continuation.asLocalOrNullIfNotUsed() : Continuation? = when (this) { + is ShareableContinuation -> localRefOrNullIfNotUsed() + else -> this +} + +internal actual fun Continuation.useLocal() : Continuation = when (this) { + is ShareableContinuation -> useRef() + else -> this +} + +internal actual fun Continuation.shareableInterceptedResumeCancellableWith(result: Result) { + this as ShareableContinuation // must have been shared + val thread = ownerThreadOrNull ?: wasUsed() + if (currentThread() == thread) { + useRef().intercepted().resumeCancellableWith(result) + } else { + thread.execute { + useRef().intercepted().resumeCancellableWith(result) + } + } +} + +internal actual fun Continuation.shareableInterceptedResumeWith(result: Result) { + this as ShareableContinuation // must have been shared + val thread = ownerThreadOrNull ?: wasUsed() + if (currentThread() == thread) { + useRef().intercepted().resumeWith(result) + } else { + thread.execute { + useRef().intercepted().resumeWith(result) + } + } +} + +internal actual fun (suspend (T) -> R).asShareable(): suspend (T) -> R = + ShareableBlock(this) + +@PublishedApi +internal actual inline fun disposeContinuation(cont: () -> Continuation<*>) { + (cont() as ShareableContinuation<*>).disposeRef() +} + +internal actual fun CancellableContinuationImpl.shareableResume(delegate: Continuation, undispatched: Boolean) { + if (delegate is ShareableContinuation) { + val thread = delegate.ownerThreadOrNull ?: delegate.wasUsed() + if (currentThread() == thread) { + resume(delegate.useRef(), undispatched) + } else { + thread.execute { + resume(delegate.useRef(), undispatched) + } + } + return + } + resume(delegate, undispatched) +} + +internal actual fun isReuseSupportedInPlatform() = false + +internal actual inline fun ArrayList.addOrUpdate(element: T, update: (ArrayList) -> Unit) { + if (isFrozen) { + val list = ArrayList(size + 1) + list.addAll(this) + list.add(element) + update(list) + } else { + add(element) + } +} + +internal actual inline fun ArrayList.addOrUpdate(index: Int, element: T, update: (ArrayList) -> Unit) { + if (isFrozen) { + val list = ArrayList(size + 1) + list.addAll(this) + list.add(index, element) + update(list) + } else { + add(index, element) + } +} + +@Suppress("NOTHING_TO_INLINE") +internal actual inline fun Any.weakRef(): Any = WeakReference(this) + +internal actual fun Any?.unweakRef(): Any? = (this as WeakReference<*>?)?.get() + +internal open class ShareableObject(obj: T) { + private val _ref = atomic?>(WorkerBoundReference(obj)) + + val ownerThreadOrNull: Thread? + get() = _ref.value?.worker?.toThread() + + fun localRef(): T { + val ref = _ref.value ?: wasUsed() + return ref.value + } + + fun localRefOrNull(): T? { + val ref = _ref.value ?: wasUsed() + if (Worker.current != ref.worker) return null + return ref.value + } + + fun localRefOrNullIfNotUsed(): T? { + val ref = _ref.value ?: return null + if (Worker.current != ref.worker) return null + return ref.value + } + + fun useRef(): T { + val ref = _ref.getAndSet(null) ?: wasUsed() + return ref.value + } + + fun disposeRef(): T? { + val ref = _ref.getAndSet(null) ?: return null + return ref.value + } + + fun wasUsed(): Nothing { + error("Ref $classSimpleName@$hexAddress was already used") + } + + override fun toString(): String { + val ref = _ref.value ?: return "Shareable[used]" + return "Shareable[${if (Worker.current == ref.worker) _ref.value.toString() else "wrong worker"}]" + } +} + +@PublishedApi +internal class ShareableContinuation( + cont: Continuation +) : ShareableObject>(cont), Continuation { + override val context: CoroutineContext = cont.context + + override fun resumeWith(result: Result) { + val thread = ownerThreadOrNull ?: wasUsed() + if (currentThread() == thread) { + useRef().resumeWith(result) + } else { + thread.execute { + useRef().resumeWith(result) + } + } + } +} + +private class ShareableDisposableHandle( + handle: DisposableHandle +) : ShareableObject(handle), DisposableHandle { + override fun dispose() { + val thread = ownerThreadOrNull ?: return + if (currentThread() == thread) { + disposeRef()?.dispose() + } else { + thread.execute { + disposeRef()?.dispose() + } + } + } +} + +private typealias Block1 = suspend (T) -> R + +// todo: SuspendFunction impl is a hack to workaround the absence of proper suspend fun implementation ability +@Suppress("SUPERTYPE_IS_SUSPEND_FUNCTION_TYPE", "INCONSISTENT_TYPE_PARAMETER_VALUES") +private class ShareableBlock( + block: Block1 +) : ShareableObject>(block), Block1 { + + override suspend fun invoke(param: T): R = useRef().invoke(param) +} diff --git a/kotlinx-coroutines-core/native/src/internal/Synchronized.kt b/kotlinx-coroutines-core/native/src/internal/Synchronized.kt index edbd3fde0c..67293aad6b 100644 --- a/kotlinx-coroutines-core/native/src/internal/Synchronized.kt +++ b/kotlinx-coroutines-core/native/src/internal/Synchronized.kt @@ -18,3 +18,6 @@ public actual typealias SynchronizedObject = kotlinx.atomicfu.locks.Synchronized */ @InternalCoroutinesApi public actual inline fun synchronized(lock: SynchronizedObject, block: () -> T): T = lock.withLock2(block) + +@OptIn(ExperimentalStdlibApi::class) +internal actual val isNativeMt: Boolean = !kotlin.native.isExperimentalMM() diff --git a/kotlinx-coroutines-core/native/test/DefaultDispatcherTest.kt b/kotlinx-coroutines-core/native/test/DefaultDispatcherTest.kt new file mode 100644 index 0000000000..d19e38c5a7 --- /dev/null +++ b/kotlinx-coroutines-core/native/test/DefaultDispatcherTest.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.test.* + +class DefaultDispatcherTest : TestBase() { + private val testThread = currentThread() + + @Test + fun testDefaultDispatcher() = runTest { + expect(1) + withContext(Dispatchers.Default) { + assertTrue(currentThread() != testThread) + expect(2) + } + assertEquals(testThread, currentThread()) + finish(3) + } +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/native/test/EventLoopTest.kt b/kotlinx-coroutines-core/native/test/EventLoopTest.kt new file mode 100644 index 0000000000..f30bf1a902 --- /dev/null +++ b/kotlinx-coroutines-core/native/test/EventLoopTest.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.coroutines.* +import kotlin.test.* + +/** + * Ensure that there are no leaks because of various delay usage. + */ +class EventLoopTest : TestBase() { + @Test + fun testDelayWait() = runTest { + expect(1) + delay(1) + finish(2) + } + + @Test + fun testDelayCancel() = runTest { + expect(1) + val job = launch(start = CoroutineStart.UNDISPATCHED) { + expect(2) + delay(100) + expectUnreached() + } + expect(3) + job.cancel() + finish(4) + } + + @Test + fun testCancellableContinuationResumeUndispatchedCancelled() = runTest { + expect(1) + var cont: CancellableContinuation? = null + val job = launch(start = CoroutineStart.UNDISPATCHED) { + expect(2) + assertFailsWith { + suspendCancellableCoroutine { cont = it } + } + expect(5) + } + expect(3) + with(cont!!) { + cancel() + // already cancelled, so nothing should happen on resumeUndispatched + (coroutineContext[ContinuationInterceptor] as CoroutineDispatcher).resumeUndispatched(Unit) + } + expect(4) + yield() + finish(6) + } +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/native/test/FreezingTest.kt b/kotlinx-coroutines-core/native/test/FreezingTest.kt new file mode 100644 index 0000000000..11d74a3de3 --- /dev/null +++ b/kotlinx-coroutines-core/native/test/FreezingTest.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.coroutines.flow.* +import kotlinx.coroutines.internal.* +import kotlin.native.concurrent.* +import kotlin.test.* + +class FreezingTest : TestBase() { + @Test + fun testFreezeWithContextOther() = runTest { + // create a mutable object referenced by this lambda + val mutable = mutableListOf() + // run a child coroutine in another thread + val result = withContext(Dispatchers.Default) { "OK" } + assertEquals("OK", result) + // ensure that objects referenced by this lambda were not frozen + assertFalse(mutable.isFrozen) + mutable.add(42) // just to be 100% sure + } + + @Test + fun testNoFreezeLaunchSame() = runTest { + if (multithreadingSupported) return@runTest + // create a mutable object referenced by this lambda + val mutable1 = mutableListOf() + // this one will get captured into the other thread's lambda + val mutable2 = mutableListOf() + val job = launch { // launch into the same context --> should not freeze + assertEquals(mutable1.isFrozen, false) + assertEquals(mutable2.isFrozen, false) + val result = withContext(Dispatchers.Default) { + assertEquals(mutable2.isFrozen, true) // was frozen now + "OK" + } + assertEquals("OK", result) + assertEquals(mutable1.isFrozen, false) + } + job.join() + assertEquals(mutable1.isFrozen, false) + mutable1.add(42) // just to be 100% sure + } + + @Test + fun testFrozenParentJob() { + val parent = Job() + parent.freeze() + val job = Job(parent) + assertTrue(job.isActive) + parent.cancel() + assertTrue(job.isCancelled) + } + + @Test + fun testStateFlowValue() = runTest { + val stateFlow = MutableStateFlow(0) + stateFlow.freeze() + stateFlow.value = 1 + } + + @Test + fun testStateFlowCollector() = runTest { + val stateFlow = MutableStateFlow(0) + stateFlow.freeze() + repeat(10) { + launch { + stateFlow.collect { + if (it == 42) cancel() + } + } + } + stateFlow.value = 42 + } + + @Test + fun testSharedFlow() = runTest { + val sharedFlow = MutableSharedFlow(0) + sharedFlow.freeze() + val job = launch { + sharedFlow.collect { + expect(it) + } + } + yield() + repeat(10) { + sharedFlow.emit(it + 1) + } + job.cancelAndJoin() + finish(11) + } + + @Test + fun testSharedFlowSubscriptionsCount() = runTest { + val sharedFlow = MutableSharedFlow(0) + sharedFlow.freeze() + val job = launch { sharedFlow.collect {} } + val subscriptions = sharedFlow.subscriptionCount.filter { count -> count > 0 }.first() + assertEquals(1, subscriptions) + job.cancelAndJoin() + } +} diff --git a/kotlinx-coroutines-core/native/test/ParkStressTest.kt b/kotlinx-coroutines-core/native/test/ParkStressTest.kt new file mode 100644 index 0000000000..6647c63e33 --- /dev/null +++ b/kotlinx-coroutines-core/native/test/ParkStressTest.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.native.concurrent.* +import kotlin.test.* +import kotlinx.coroutines.* +import kotlinx.coroutines.exceptions.* + +private const val timeoutMicroseconds = Long.MAX_VALUE / 1000L // too long. +private const val nTasks = 10_000 // repeat test + +/** + * This stress test ensures that Worker.park correctly wakes up. + */ +class ParkStressTest { + @Test + fun testPark() { + val worker = Worker.start() + worker.execute(TransferMode.SAFE, {}) { + // process nTasks + while (TaskCounter.counter < nTasks) { + randomWait() + val ok = Worker.current.park(timeoutMicroseconds, process = true) + assertTrue(ok, "Must have processed a task") + } + assertEquals(nTasks, TaskCounter.counter) + } + // submit nTasks + repeat(nTasks) { index -> + randomWait() + val operation: () -> Unit = { + TaskCounter.counter++ + } + operation.freeze() + worker.executeAfter(0, operation) + } + // shutdown worker + worker.requestTermination().result // block until termination + } +} + +@ThreadLocal +private object TaskCounter { + var counter = 0 +} + diff --git a/kotlinx-coroutines-core/native/test/TestBase.kt b/kotlinx-coroutines-core/native/test/TestBase.kt index 6fef4752a8..f44a032aec 100644 --- a/kotlinx-coroutines-core/native/test/TestBase.kt +++ b/kotlinx-coroutines-core/native/test/TestBase.kt @@ -1,5 +1,5 @@ /* - * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines @@ -17,9 +17,9 @@ public actual typealias TestResult = Unit public actual open class TestBase actual constructor() { public actual val isBoundByJsTestTimeout = false - private var actionIndex = atomic(0) - private var finished = atomic(false) - private var error: Throwable? = null + private val actionIndex = atomic(0) + private val finished = atomic(false) + private val error = atomic(null) /** * Throws [IllegalStateException] like `error` in stdlib, but also ensures that the test will not @@ -28,13 +28,14 @@ public actual open class TestBase actual constructor() { @Suppress("ACTUAL_FUNCTION_WITH_DEFAULT_ARGUMENTS") public actual fun error(message: Any, cause: Throwable? = null): Nothing { val exception = IllegalStateException(message.toString(), cause) - if (error == null) error = exception + error.compareAndSet(null, exception) throw exception } private fun printError(message: String, cause: Throwable) { - if (error == null) error = cause - println("$message: $cause") + error.compareAndSet(null, cause) + println(message) + cause.printStackTrace() } /** @@ -80,30 +81,30 @@ public actual open class TestBase actual constructor() { unhandled: List<(Throwable) -> Boolean> = emptyList(), block: suspend CoroutineScope.() -> Unit ): TestResult { - var exCount = 0 - var ex: Throwable? = null + val exCount = atomic(0) + val ex = atomic(null) try { runBlocking(block = block, context = CoroutineExceptionHandler { _, e -> if (e is CancellationException) return@CoroutineExceptionHandler // are ignored - exCount++ + val result = exCount.incrementAndGet() when { - exCount > unhandled.size -> - printError("Too many unhandled exceptions $exCount, expected ${unhandled.size}, got: $e", e) - !unhandled[exCount - 1](e) -> + result > unhandled.size -> + printError("Too many unhandled exceptions $result, expected ${unhandled.size}, got: $e", e) + !unhandled[result - 1](e) -> printError("Unhandled exception was unexpected: $e", e) } }) } catch (e: Throwable) { - ex = e + ex.value = e if (expected != null) { if (!expected(e)) error("Unexpected exception: $e", e) } else throw e } finally { - if (ex == null && expected != null) error("Exception was expected but none produced") + if (ex.value == null && expected != null) error("Exception was expected but none produced") } - if (exCount < unhandled.size) + if (exCount.value < unhandled.size) error("Too few unhandled exceptions $exCount, expected ${unhandled.size}") } } diff --git a/kotlinx-coroutines-core/native/test/WithTimeoutNativeTest.kt b/kotlinx-coroutines-core/native/test/WithTimeoutNativeTest.kt new file mode 100644 index 0000000000..9a1f0bd598 --- /dev/null +++ b/kotlinx-coroutines-core/native/test/WithTimeoutNativeTest.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.test.* +import kotlin.time.* +import kotlin.native.concurrent.freeze + +class WithTimeoutNativeTest { + @OptIn(ExperimentalTime::class) + @Test + fun `withTimeout should not raise an exception when parent job is frozen`() = runBlocking { + this.coroutineContext[Job.Key]!!.freeze() + try { + withTimeout(Duration.seconds(Int.MAX_VALUE)) {} + } catch (error: Throwable) { + fail("withTimeout has raised an exception.", error) + } + } +} diff --git a/kotlinx-coroutines-core/native/test/WorkerDispatcherTest.kt b/kotlinx-coroutines-core/native/test/WorkerDispatcherTest.kt new file mode 100644 index 0000000000..d39ac0b4d2 --- /dev/null +++ b/kotlinx-coroutines-core/native/test/WorkerDispatcherTest.kt @@ -0,0 +1,348 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.coroutines.channels.* +import kotlinx.coroutines.flow.* +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.sync.* +import kotlin.native.concurrent.* +import kotlin.test.* + +class WorkerDispatcherTest : TestBase() { + private val dispatcher = newSingleThreadContext("WorkerCoroutineDispatcherTest") + private val mainThread = currentThread() + + private fun runTest(block: suspend CoroutineScope.() -> Unit) { + if (multithreadingSupported) return + else super.runTest(null, emptyList(), block) + } + + @AfterTest + fun tearDown() { + dispatcher.close() + } + + @Test + fun testWithContext() = runTest { + val atomic = AtomicInt(0) // can be captured & shared + expect(1) + val result = withContext(dispatcher) { + expect(2) + assertEquals(dispatcher.thread, currentThread()) + atomic.value = 42 + "OK" + } + assertEquals(mainThread, currentThread()) + assertEquals("OK", result) + assertEquals(42, atomic.value) + finish(3) + } + + @Test + fun testLaunchJoin() = runTest { + val atomic = AtomicInt(0) // can be captured & shared + expect(1) + val job = launch(dispatcher) { + assertEquals(dispatcher.thread, currentThread()) + atomic.value = 42 + } + job.join() + assertEquals(mainThread, currentThread()) + assertEquals(42, atomic.value) + finish(2) + } + + @Test + fun testLaunchLazyJoin() = runTest { + expect(1) + val job = launch(dispatcher, start = CoroutineStart.LAZY) { + expect(3) + assertEquals(dispatcher.thread, currentThread()) + } + expect(2) + job.join() // lazy start here + finish(4) + } + + @Test + fun testAsyncAwait() = runTest { + val atomic = AtomicInt(0) // can be captured & shared + expect(1) + val deferred = async(dispatcher) { + assertEquals(dispatcher.thread, currentThread()) + atomic.value = 42 + "OK" + } + val result = deferred.await() + assertEquals(mainThread, currentThread()) + assertEquals("OK", result) + assertEquals(42, atomic.value) + finish(2) + } + + @Test + fun testAsyncLazyAwait() = runTest { + expect(1) + val deferred = async(dispatcher, start = CoroutineStart.LAZY) { + expect(3) + assertEquals(dispatcher.thread, currentThread()) + "OK" + } + expect(2) + val result = deferred.await() // lazy start here + assertEquals("OK", result) + finish(4) + } + + @Test + fun testProduceConsumeRendezvous() = checkProduceConsume(Channel.RENDEZVOUS) + + @Test + fun testProduceConsumeUnlimited() = checkProduceConsume(Channel.UNLIMITED) + + @Test + fun testProduceConsumeBuffered() = checkProduceConsume(10) + + private fun checkProduceConsume(capacity: Int) { + runTest { + val atomic = AtomicInt(0) // can be captured & shared + expect(1) + val channel = produce(dispatcher, capacity) { + assertEquals(dispatcher.thread, currentThread()) + atomic.value = 42 + expect(2) + send(Data("A")) + send(Data("B")) + } + val result1 = channel.receive() + expect(3) + assertEquals(mainThread, currentThread()) + assertEquals("A", result1.s) + assertTrue(result1.isFrozen) + assertEquals(42, atomic.value) + val result2 = channel.receive() + assertEquals("B", result2.s) + assertEquals(null, channel.receiveCatching().getOrNull()) // must try to receive the last one to dispose memory + finish(4) + } + } + + @Test + fun testChannelIterator() = runTest { + expect(1) + val channel = RendezvousChannel(null) + launch(dispatcher) { + channel.send(1) + channel.send(2) + channel.close() + } + var expected = 1 + for (x in channel) { + assertEquals(expected++, x) + } + finish(2) + } + + @Test + fun testArrayBroadcast() = runTest { + expect(1) + val broadcast = BroadcastChannel(10) + val sub = broadcast.openSubscription() + launch(dispatcher) { + assertEquals(dispatcher.thread, currentThread()) + expect(2) + broadcast.send(Data("A")) + broadcast.send(Data("B")) + } + val result1 = sub.receive() + expect(3) + assertEquals(mainThread, currentThread()) + assertEquals("A", result1.s) + assertTrue(result1.isFrozen) + val result2 = sub.receive() + assertEquals("B", result2.s) + sub.cancel() + broadcast.close() // dispose memory + finish(4) + } + + @Test + fun testConflatedBroadcast() = runTest { + expect(1) + val latch = Channel() + val broadcast = ConflatedBroadcastChannel() + val sub = broadcast.openSubscription() + launch(dispatcher) { + assertEquals(dispatcher.thread, currentThread()) + expect(2) + broadcast.send(Data("A")) + latch.receive() + expect(4) + broadcast.send(Data("B")) + } + val result1 = sub.receive() + expect(3) + assertEquals(mainThread, currentThread()) + assertEquals("A", result1.s) + assertTrue(result1.isFrozen) + latch.send(Unit) + val result2 = sub.receive() + assertEquals("B", result2.s) + sub.cancel() + broadcast.close() // dispose memory + latch.close() // dispose memory + finish(5) + } + + @Test + fun testFlowOn() = runTest { + expect(1) + val flow = flow { + expect(3) + assertEquals(dispatcher.thread, currentThread()) + emit(Data("A")) + emit(Data("B")) + }.flowOn(dispatcher) + expect(2) + val result = flow.toList() + assertEquals(listOf(Data("A"), Data("B")), result) + assertTrue(result.all { it.isFrozen }) + finish(4) + } + + @Test + fun testWithContextDelay() = runTest { + expect(1) + withContext(dispatcher) { + expect(2) + delay(10) + assertEquals(dispatcher.thread, currentThread()) + expect(3) + } + finish(4) + } + + @Test + fun testWithTimeoutAroundWithContextNoTimeout() = runTest { + expect(1) + withTimeout(1000) { + withContext(dispatcher) { + expect(2) + } + } + finish(3) + } + + @Test + fun testWithTimeoutAroundWithContextTimedOut() = runTest { + expect(1) + assertFailsWith { + withTimeout(100) { + withContext(dispatcher) { + expect(2) + delay(1000) + } + } + } + finish(3) + } + + @Test // Can sometimes hang tho + fun testMutexStress() = runTest { + expect(1) + val mutex = Mutex() + val atomic = AtomicInt(0) + val n = 100 + val k = 239 // mutliplier + val job = launch(dispatcher) { + repeat(n) { + mutex.withLock { + atomic.value = atomic.value + 1 // unsafe mutation but under mutex + } + } + } + // concurrently mutate + repeat(n) { + mutex.withLock { + atomic.value = atomic.value + k + } + } + // join job + job.join() + assertEquals((k + 1) * n, atomic.value) + finish(2) + } + + @Test + fun testSemaphoreStress() = runTest { + expect(1) + val semaphore = Semaphore(1) + val atomic = AtomicInt(0) + val n = 100 + val k = 239 // mutliplier + val job = launch(dispatcher) { + repeat(n) { + semaphore.withPermit { + atomic.value = atomic.value + 1 // unsafe mutation but under mutex + } + } + } + // concurrently mutate + repeat(n) { + semaphore.withPermit { + atomic.value = atomic.value + k + } + } + // join job + job.join() + assertEquals((k + 1) * n, atomic.value) + finish(2) + } + + @Test + fun testBroadcastAsFlow() = runTest { + expect(1) + withContext(dispatcher) { + expect(2) + broadcast { + expect(3) + send("OK") + }.asFlow().collect { + expect(4) + assertEquals("OK", it) + } + expect(5) + } + finish(6) + } + + @Test + fun testAwaitAll() = runTest { + expect(1) + val d1 = async(dispatcher) { + "A" + } + val d2 = async(dispatcher) { + "B" + } + assertEquals("AB", awaitAll(d1, d2).joinToString("")) + finish(2) + } + + @Test + fun testEnsureNeverFrozenWithContext() = runTest { + expect(1) + val x = Data("OK") + x.ensureNeverFrozen() + assertFailsWith { + val s = withContext(dispatcher) { x.s } + println(s) // does not actually execute + } + finish(2) + } + + private data class Data(val s: String) +} diff --git a/kotlinx-coroutines-core/native/test/WorkerTest.kt b/kotlinx-coroutines-core/native/test/WorkerTest.kt index 8252ca656a..d9761dedd1 100644 --- a/kotlinx-coroutines-core/native/test/WorkerTest.kt +++ b/kotlinx-coroutines-core/native/test/WorkerTest.kt @@ -9,22 +9,26 @@ import kotlin.native.concurrent.* import kotlin.test.* class WorkerTest : TestBase() { + val worker = Worker.start() + + @AfterTest + fun tearDown() { + worker.requestTermination().result + } @Test fun testLaunchInWorker() { - val worker = Worker.start() worker.execute(TransferMode.SAFE, { }) { runBlocking { launch { }.join() delay(1) + println("Done") } }.result - worker.requestTermination() } @Test fun testLaunchInWorkerThroughGlobalScope() { - val worker = Worker.start() worker.execute(TransferMode.SAFE, { }) { runBlocking { CoroutineScope(EmptyCoroutineContext).launch { @@ -32,6 +36,5 @@ class WorkerTest : TestBase() { }.join() } }.result - worker.requestTermination() } } diff --git a/kotlinx-coroutines-core/native/test/internal/LinkedListTest.kt b/kotlinx-coroutines-core/native/test/internal/LinkedListTest.kt deleted file mode 100644 index 44ddf471d7..0000000000 --- a/kotlinx-coroutines-core/native/test/internal/LinkedListTest.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.internal - -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class LinkedListTest { - data class IntNode(val i: Int) : LockFreeLinkedListNode() - - @Test - fun testSimpleAddLastRemove() { - val list = LockFreeLinkedListHead() - assertContents(list) - val n1 = IntNode(1).apply { list.addLast(this) } - assertContents(list, 1) - val n2 = IntNode(2).apply { list.addLast(this) } - assertContents(list, 1, 2) - val n3 = IntNode(3).apply { list.addLast(this) } - assertContents(list, 1, 2, 3) - val n4 = IntNode(4).apply { list.addLast(this) } - assertContents(list, 1, 2, 3, 4) - assertTrue(n1.remove()) - assertContents(list, 2, 3, 4) - assertTrue(n3.remove()) - assertContents(list, 2, 4) - assertTrue(n4.remove()) - assertContents(list, 2) - assertTrue(n2.remove()) - assertFalse(n2.remove()) - assertContents(list) - } - - private fun assertContents(list: LockFreeLinkedListHead, vararg expected: Int) { - val n = expected.size - val actual = IntArray(n) - var index = 0 - list.forEach { actual[index++] = it.i } - assertEquals(n, index) - for (i in 0 until n) assertEquals(expected[i], actual[i], "item i") - assertEquals(expected.isEmpty(), list.isEmpty) - } -} diff --git a/kotlinx-coroutines-core/nativeDarwin/src/Dispatchers.kt b/kotlinx-coroutines-core/nativeDarwin/src/Dispatchers.kt index ace20422f6..ed556b2cc9 100644 --- a/kotlinx-coroutines-core/nativeDarwin/src/Dispatchers.kt +++ b/kotlinx-coroutines-core/nativeDarwin/src/Dispatchers.kt @@ -5,7 +5,6 @@ package kotlinx.coroutines import kotlinx.cinterop.* -import kotlinx.coroutines.internal.* import platform.CoreFoundation.* import platform.darwin.* import kotlin.coroutines.* @@ -15,34 +14,25 @@ import kotlin.native.internal.NativePtr internal fun isMainThread(): Boolean = CFRunLoopGetCurrent() == CFRunLoopGetMain() internal actual fun createMainDispatcher(default: CoroutineDispatcher): MainCoroutineDispatcher = - if (multithreadingSupported) DarwinMainDispatcher(false) else OldMainDispatcher(Dispatchers.Default) - -internal actual fun createDefaultDispatcher(): CoroutineDispatcher = DarwinGlobalQueueDispatcher - -private object DarwinGlobalQueueDispatcher : CoroutineDispatcher() { - override fun dispatch(context: CoroutineContext, block: Runnable) { - autoreleasepool { - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT.convert(), 0)) { - block.run() - } - } - } -} + DarwinMainDispatcher(false) +@Suppress("EXPERIMENTAL_UNSIGNED_LITERALS") private class DarwinMainDispatcher( private val invokeImmediately: Boolean -) : MainCoroutineDispatcher(), Delay { +) : MainCoroutineDispatcher(), Delay, ThreadBoundInterceptor { + override val thread + get() = mainThread override val immediate: MainCoroutineDispatcher = if (invokeImmediately) this else DarwinMainDispatcher(true) + init { freeze() } + override fun isDispatchNeeded(context: CoroutineContext): Boolean = !(invokeImmediately && isMainThread()) override fun dispatch(context: CoroutineContext, block: Runnable) { - autoreleasepool { - dispatch_async(dispatch_get_main_queue()) { - block.run() - } + dispatch_async(dispatch_get_main_queue()) { + block.run() } } @@ -70,16 +60,22 @@ private class DarwinMainDispatcher( "MainDispatcher${ if(invokeImmediately) "[immediate]" else "" }" } -private typealias TimerBlock = (CFRunLoopTimerRef?) -> Unit +internal typealias TimerBlock = (CFRunLoopTimerRef?) -> Unit +@SharedImmutable private val TIMER_NEW = NativePtr.NULL + +@SharedImmutable private val TIMER_DISPOSED = NativePtr.NULL.plus(1) private class Timer : DisposableHandle { private val ref = AtomicNativePtr(TIMER_NEW) + init { freeze() } + fun start(timeMillis: Long, timerBlock: TimerBlock) { val fireDate = CFAbsoluteTimeGetCurrent() + timeMillis / 1000.0 + @Suppress("EXPERIMENTAL_UNSIGNED_LITERALS") val timer = CFRunLoopTimerCreateWithHandler(null, fireDate, 0.0, 0u, 0, timerBlock) CFRunLoopAddTimer(CFRunLoopGetMain(), timer, kCFRunLoopCommonModes) if (!ref.compareAndSet(TIMER_NEW, timer.rawValue)) { @@ -104,5 +100,3 @@ private class Timer : DisposableHandle { CFRelease(timer) } } - -internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit): Unit = autoreleasepool { block() } diff --git a/kotlinx-coroutines-core/nativeDarwin/src/EventLoop.kt b/kotlinx-coroutines-core/nativeDarwin/src/EventLoop.kt new file mode 100644 index 0000000000..00a155de17 --- /dev/null +++ b/kotlinx-coroutines-core/nativeDarwin/src/EventLoop.kt @@ -0,0 +1,9 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.cinterop.* + +internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit): Unit = autoreleasepool { block() } diff --git a/kotlinx-coroutines-core/nativeDarwin/src/Thread.kt b/kotlinx-coroutines-core/nativeDarwin/src/Thread.kt new file mode 100644 index 0000000000..9ebf68d2ce --- /dev/null +++ b/kotlinx-coroutines-core/nativeDarwin/src/Thread.kt @@ -0,0 +1,73 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.atomicfu.* +import kotlinx.cinterop.* +import platform.CoreFoundation.* +import platform.darwin.* +import kotlin.native.concurrent.* +import kotlin.native.* +import kotlin.native.concurrent.freeze as stdlibFreeze + +/** + * Initializes the main thread. Must be called from the main thread if the application's interaction + * with Kotlin runtime and coroutines API otherwise starts from background threads. + */ +@ExperimentalCoroutinesApi +public fun initMainThread() { + getOrCreateMainThread() +} + +internal actual fun initCurrentThread(): Thread = + if (isMainThread()) mainThread else WorkerThread() + +internal actual fun Worker.toThread(): Thread = + if (this == mainThread.worker) mainThread else WorkerThread(this) + +@SharedImmutable +private val _mainThread = AtomicReference(null) + +@OptIn(ExperimentalStdlibApi::class) +@EagerInitialization +internal val mainThread: MainThread get() = _mainThread.value ?: getOrCreateMainThread() + +private fun getOrCreateMainThread(): MainThread { + require(isMainThread()) { + "Coroutines must be initialized from the main thread: call 'initMainThread' from the main thread first" + } + _mainThread.value?.let { return it } + return MainThread().also { _mainThread.value = it } +} + +internal class MainThread : WorkerThread() { + private val posted = atomic(false) + + private val processQueueBlock: dispatch_block_t = { + posted.value = false // next execute will post a fresh task + while (worker.processQueue()) { /* process all */ } + } + + init { stdlibFreeze() } + + override fun execute(block: () -> Unit) { + super.execute(block) + // post to main queue if needed + if (posted.compareAndSet(false, true)) { + dispatch_async(dispatch_get_main_queue(), processQueueBlock) + } + } + + fun shutdown() { + // Cleanup posted processQueueBlock + execute { + CFRunLoopStop(CFRunLoopGetCurrent()) + } + CFRunLoopRun() + assert(!posted.value) // nothing else should have been posted + } + + override fun toString(): String = "MainThread" +} diff --git a/kotlinx-coroutines-core/nativeDarwin/test/AutoreleaseLeakTest.kt b/kotlinx-coroutines-core/nativeDarwin/test/AutoreleaseLeakTest.kt new file mode 100644 index 0000000000..af7fd4f437 --- /dev/null +++ b/kotlinx-coroutines-core/nativeDarwin/test/AutoreleaseLeakTest.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.native.ref.* +import kotlin.native.concurrent.* +import kotlin.native.internal.* +import platform.Foundation.* +import platform.darwin.NSObject +import kotlinx.cinterop.* +import kotlinx.coroutines.internal.* +import kotlin.test.* + +class AutoreleaseLeakTest : TestBase() { + private val testThread = currentThread() + + @Test + fun testObjCAutorelease() { + if (multithreadingSupported) return + val weakRef = AtomicReference?>(null) + + runTest { + withContext(Dispatchers.Default) { + val job = launch { + assertNotEquals(testThread, currentThread()) + val objcObj = NSObject() + weakRef.value = WeakReference(objcObj).freeze() + + // Emulate an autorelease return value in native code. + objc_retainAutoreleaseReturnValue(objcObj.objcPtr()) + } + job.join() + GC.collect() + } + + assertNotNull(weakRef.value) + assertNull(weakRef.value?.value) + } + } +} diff --git a/kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt b/kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt index 78ed765967..9c91407f73 100644 --- a/kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt +++ b/kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt @@ -7,10 +7,12 @@ package kotlinx.coroutines import platform.CoreFoundation.* import kotlin.native.concurrent.* import kotlin.native.internal.test.* +import kotlin.native.Platform import kotlin.system.* // This is a separate entry point for tests in background fun mainBackground(args: Array) { + mainThread // Force init val worker = Worker.start(name = "main-background") worker.execute(TransferMode.SAFE, { args.freeze() }) { val result = testLauncherEntryPoint(it) @@ -22,7 +24,11 @@ fun mainBackground(args: Array) { // This is a separate entry point for tests with leak checker fun mainNoExit(args: Array) { + mainThread // Force init + Platform.isMemoryLeakCheckerActive = true workerMain { // autoreleasepool to make sure interop objects are properly freed testLauncherEntryPoint(args) + mainThread.shutdown() + DefaultDispatcher.shutdown() } -} \ No newline at end of file +} diff --git a/kotlinx-coroutines-core/nativeDarwin/test/MainDispatcherTest.kt b/kotlinx-coroutines-core/nativeDarwin/test/MainDispatcherTest.kt index d460bd6e6e..0431c4e257 100644 --- a/kotlinx-coroutines-core/nativeDarwin/test/MainDispatcherTest.kt +++ b/kotlinx-coroutines-core/nativeDarwin/test/MainDispatcherTest.kt @@ -1,130 +1,132 @@ /* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines -import kotlinx.coroutines.internal.* -import platform.CoreFoundation.* -import platform.darwin.* import kotlin.coroutines.* import kotlin.test.* class MainDispatcherTest : TestBase() { - - private fun isMainThread(): Boolean = CFRunLoopGetCurrent() == CFRunLoopGetMain() - private fun canTestMainDispatcher() = !isMainThread() && multithreadingSupported - - private fun runTestNotOnMainDispatcher(block: suspend CoroutineScope.() -> Unit) { - // skip if already on the main thread, run blocking doesn't really work well with that - if (!canTestMainDispatcher()) return - runTest(block = block) - } - - @Test - fun testDispatchNecessityCheckWithMainImmediateDispatcher() = runTestNotOnMainDispatcher { - val main = Dispatchers.Main.immediate - assertTrue(main.isDispatchNeeded(EmptyCoroutineContext)) - withContext(Dispatchers.Default) { - assertTrue(main.isDispatchNeeded(EmptyCoroutineContext)) - withContext(Dispatchers.Main) { - assertFalse(main.isDispatchNeeded(EmptyCoroutineContext)) - } - assertTrue(main.isDispatchNeeded(EmptyCoroutineContext)) - } - } + private val testThread = currentThread() @Test - fun testWithContext() = runTestNotOnMainDispatcher { - expect(1) - assertFalse(isMainThread()) - withContext(Dispatchers.Main) { - assertTrue(isMainThread()) - expect(2) + fun testDispatchNecessityCheckWithMainImmediateDispatcher() { + runTest { + val immediate = Dispatchers.Main.immediate + val needsDispatch = testThread != mainThread + assertEquals(needsDispatch, immediate.isDispatchNeeded(EmptyCoroutineContext)) } - assertFalse(isMainThread()) - finish(3) } @Test - fun testWithContextDelay() = runTestNotOnMainDispatcher { - expect(1) - withContext(Dispatchers.Main) { - assertTrue(isMainThread()) - expect(2) - delay(100) - assertTrue(isMainThread()) - expect(3) + fun testWithContext() { + if (testThread == mainThread) return // skip if already on the main thread + runTest { + expect(1) + withContext(Dispatchers.Main) { + assertEquals(mainThread, currentThread()) + expect(2) + } + assertEquals(testThread, currentThread()) + finish(3) } - assertFalse(isMainThread()) - finish(4) } @Test - fun testWithTimeoutContextDelayNoTimeout() = runTestNotOnMainDispatcher { - expect(1) - withTimeout(1000) { + fun testWithContextDelay() { + if (testThread == mainThread) return // skip if already on the main thread + runTest { + expect(1) withContext(Dispatchers.Main) { - assertTrue(isMainThread()) + assertEquals(mainThread, currentThread()) expect(2) delay(100) - assertTrue(isMainThread()) + assertEquals(mainThread, currentThread()) expect(3) } + assertEquals(testThread, currentThread()) + finish(4) } - assertFalse(isMainThread()) - finish(4) } @Test - fun testWithTimeoutContextDelayTimeout() = runTestNotOnMainDispatcher { - expect(1) - assertFailsWith { - withTimeout(100) { + fun testWithTimeoutContextDelayNoTimeout() { + if (testThread == mainThread) return // skip if already on the main thread + runTest { + expect(1) + withTimeout(1000) { withContext(Dispatchers.Main) { - assertTrue(isMainThread()) + assertEquals(mainThread, currentThread()) expect(2) - delay(1000) - expectUnreached() + delay(100) + assertEquals(mainThread, currentThread()) + expect(3) } } - expectUnreached() + assertEquals(testThread, currentThread()) + finish(4) } - assertFalse(isMainThread()) - finish(3) } @Test - fun testWithContextTimeoutDelayNoTimeout() = runTestNotOnMainDispatcher { - expect(1) - withContext(Dispatchers.Main) { - withTimeout(1000) { - assertTrue(isMainThread()) - expect(2) - delay(100) - assertTrue(isMainThread()) - expect(3) + fun testWithTimeoutContextDelayTimeout() { + if (testThread == mainThread) return // skip if already on the main thread + runTest { + expect(1) + assertFailsWith { + withTimeout(100) { + withContext(Dispatchers.Main) { + assertEquals(mainThread, currentThread()) + expect(2) + delay(1000) + expectUnreached() + } + } + expectUnreached() } + assertEquals(testThread, currentThread()) + finish(3) } - assertFalse(isMainThread()) - finish(4) } @Test - fun testWithContextTimeoutDelayTimeout() = runTestNotOnMainDispatcher { - expect(1) - assertFailsWith { + fun testWithContextTimeoutDelayNoTimeout() { + if (testThread == mainThread) return // skip if already on the main thread + runTest { + expect(1) withContext(Dispatchers.Main) { - withTimeout(100) { - assertTrue(isMainThread()) + withTimeout(1000) { + assertEquals(mainThread, currentThread()) expect(2) - delay(1000) - expectUnreached() + delay(100) + assertEquals(mainThread, currentThread()) + expect(3) + } + } + assertEquals(testThread, currentThread()) + finish(4) + } + } + + @Test + fun testWithContextTimeoutDelayTimeout() { + if (testThread == mainThread) return // skip if already on the main thread + runTest { + expect(1) + assertFailsWith { + withContext(Dispatchers.Main) { + withTimeout(100) { + assertEquals(mainThread, currentThread()) + expect(2) + delay(1000) + expectUnreached() + } } + expectUnreached() } - expectUnreached() + assertEquals(testThread, currentThread()) + finish(3) } - assertFalse(isMainThread()) - finish(3) } } diff --git a/kotlinx-coroutines-core/nativeOther/src/Dispatchers.kt b/kotlinx-coroutines-core/nativeOther/src/Dispatchers.kt index 517190d0a3..b3c2dad03f 100644 --- a/kotlinx-coroutines-core/nativeOther/src/Dispatchers.kt +++ b/kotlinx-coroutines-core/nativeOther/src/Dispatchers.kt @@ -4,24 +4,12 @@ package kotlinx.coroutines +import kotlinx.coroutines.internal.multithreadingSupported import kotlin.coroutines.* internal actual fun createMainDispatcher(default: CoroutineDispatcher): MainCoroutineDispatcher = MissingMainDispatcher -internal actual fun createDefaultDispatcher(): CoroutineDispatcher = DefaultDispatcher - -private object DefaultDispatcher : CoroutineDispatcher() { - - // Delegated, so users won't be able to downcast and call 'close' - // The precise number of threads cannot be obtained until KT-48179 is implemented, 4 is just "good enough" number. - private val ctx = newFixedThreadPoolContext(4, "Dispatchers.Default") - - override fun dispatch(context: CoroutineContext, block: Runnable) { - ctx.dispatch(context, block) - } -} - private object MissingMainDispatcher : MainCoroutineDispatcher() { override val immediate: MainCoroutineDispatcher get() = notImplemented() @@ -31,5 +19,3 @@ private object MissingMainDispatcher : MainCoroutineDispatcher() { private fun notImplemented(): Nothing = TODO("Dispatchers.Main is missing on the current platform") } - -internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit) = block() diff --git a/kotlinx-coroutines-core/nativeOther/src/EventLoop.kt b/kotlinx-coroutines-core/nativeOther/src/EventLoop.kt new file mode 100644 index 0000000000..1dcc66ce1e --- /dev/null +++ b/kotlinx-coroutines-core/nativeOther/src/EventLoop.kt @@ -0,0 +1,7 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +internal actual inline fun platformAutoreleasePool(crossinline block: () -> Unit): Unit = block() diff --git a/kotlinx-coroutines-core/nativeOther/src/Thread.kt b/kotlinx-coroutines-core/nativeOther/src/Thread.kt new file mode 100644 index 0000000000..0034b3824f --- /dev/null +++ b/kotlinx-coroutines-core/nativeOther/src/Thread.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.native.concurrent.* + +internal actual fun initCurrentThread(): Thread = WorkerThread() + +internal actual fun Worker.toThread(): Thread = WorkerThread(this) diff --git a/kotlinx-coroutines-core/nativeOther/test/Launcher.kt b/kotlinx-coroutines-core/nativeOther/test/Launcher.kt index feddd4c097..ca65228435 100644 --- a/kotlinx-coroutines-core/nativeOther/test/Launcher.kt +++ b/kotlinx-coroutines-core/nativeOther/test/Launcher.kt @@ -6,6 +6,7 @@ package kotlinx.coroutines import kotlin.native.concurrent.* import kotlin.native.internal.test.* +import kotlin.native.Platform import kotlin.system.* // This is a separate entry point for tests in background @@ -19,5 +20,7 @@ fun mainBackground(args: Array) { // This is a separate entry point for tests with leak checker fun mainNoExit(args: Array) { + Platform.isMemoryLeakCheckerActive = true testLauncherEntryPoint(args) + DefaultDispatcher.shutdown() } \ No newline at end of file diff --git a/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt b/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt index fd0279123f..cab2870165 100644 --- a/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt +++ b/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines.debug @@ -52,15 +52,16 @@ class CoroutinesDumpTest : DebugTestBase() { "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.activeMethod(CoroutinesDumpTest.kt:133)\n" + "\tat kotlinx.coroutines.debug.CoroutinesDumpTest\$testRunningCoroutine\$1$deferred\$1.invokeSuspend(CoroutinesDumpTest.kt:41)\n" + "\t(Coroutine creation stacktrace)\n" + - "\tat kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt:116)\n" + - "\tat kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt:23)\n" + - "\tat kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt:99)\n" + - "\tat kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt:148)\n" + - "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.async(Builders.common.kt)\n" + - "\tat kotlinx.coroutines.BuildersKt.async(Unknown Source)\n" + - "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.async\$default(Builders.common.kt)\n" + - "\tat kotlinx.coroutines.BuildersKt.async\$default(Unknown Source)\n" + - "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.testRunningCoroutine(CoroutinesDumpTest.kt:49)", + "\tat kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt)\n" + + "\tat kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt)\n" + + "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.startCoroutineImpl(Builders.common.kt)\n" + + "\tat kotlinx.coroutines.BuildersKt.startCoroutineImpl(Unknown Source)\n" + + "\tat kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt)\n" + + "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.async(Builders.common.kt)\n" + + "\tat kotlinx.coroutines.BuildersKt.async(Unknown Source)\n" + + "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.async\$default(Builders.common.kt)\n" + + "\tat kotlinx.coroutines.BuildersKt.async\$default(Unknown Source)\n" + + "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.testRunningCoroutine(CoroutinesDumpTest.kt)", ignoredCoroutine = "BlockingCoroutine" ) { deferred.cancel() @@ -78,19 +79,20 @@ class CoroutinesDumpTest : DebugTestBase() { awaitCoroutine() verifyDump( "Coroutine \"coroutine#1\":DeferredCoroutine{Active}@1e4a7dd4, state: RUNNING\n" + - "\tat java.lang.Thread.sleep(Native Method)\n" + - "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.nestedActiveMethod(CoroutinesDumpTest.kt:111)\n" + - "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.activeMethod(CoroutinesDumpTest.kt:106)\n" + - "\t(Coroutine creation stacktrace)\n" + - "\tat kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt:116)\n" + - "\tat kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt:23)\n" + - "\tat kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt:99)\n" + - "\tat kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt:148)\n" + + "\tat java.lang.Thread.sleep(Native Method)\n" + + "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.nestedActiveMethod(CoroutinesDumpTest.kt:111)\n" + + "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.activeMethod(CoroutinesDumpTest.kt:106)\n" + + "\t(Coroutine creation stacktrace)\n" + + "\tat kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt)\n" + + "\tat kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt)\n" + + "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.startCoroutineImpl(Builders.common.kt)\n" + + "\tat kotlinx.coroutines.BuildersKt.startCoroutineImpl(Unknown Source)\n" + + "\tat kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt)\n" + "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.async(Builders.common.kt)\n" + "\tat kotlinx.coroutines.BuildersKt.async(Unknown Source)\n" + "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.async\$default(Builders.common.kt)\n" + "\tat kotlinx.coroutines.BuildersKt.async\$default(Unknown Source)\n" + - "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.testRunningCoroutineWithSuspensionPoint(CoroutinesDumpTest.kt:71)", + "\tat kotlinx.coroutines.debug.CoroutinesDumpTest.testRunningCoroutineWithSuspensionPoint(CoroutinesDumpTest.kt)", ignoredCoroutine = "BlockingCoroutine" ) { deferred.cancel() @@ -113,24 +115,18 @@ class CoroutinesDumpTest : DebugTestBase() { deferred.cancel() coroutineThread!!.interrupt() - val expected = - "kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt)\n" + - "kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt)\n" + - "kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable\$default(Cancellable.kt)\n" + - "kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt)\n" + - "kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt)\n" + - "kotlinx.coroutines.BuildersKt__Builders_commonKt.async(Builders.common.kt)\n" + - "kotlinx.coroutines.BuildersKt.async(Unknown Source)\n" + - "kotlinx.coroutines.BuildersKt__Builders_commonKt.async\$default(Builders.common.kt)\n" + - "kotlinx.coroutines.BuildersKt.async\$default(Unknown Source)\n" + - "kotlinx.coroutines.debug.CoroutinesDumpTest\$testCreationStackTrace\$1.invokeSuspend(CoroutinesDumpTest.kt)" - if (!result.startsWith(expected)) { - println("=== Actual result") - println(result) - error("Does not start with expected lines") - } - + ("kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt)\n" + + "kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt)\n" + + "kotlinx.coroutines.BuildersKt__Builders_commonKt.startCoroutineImpl(Builders.common.kt)\n" + + "kotlinx.coroutines.BuildersKt.startCoroutineImpl(Unknown Source)\n" + + "kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt)\n" + + "kotlinx.coroutines.BuildersKt__Builders_commonKt.async(Builders.common.kt)\n" + + "kotlinx.coroutines.BuildersKt.async(Unknown Source)\n" + + "kotlinx.coroutines.BuildersKt__Builders_commonKt.async\$default(Builders.common.kt)\n" + + "kotlinx.coroutines.BuildersKt.async\$default(Unknown Source)\n" + + "kotlinx.coroutines.debug.CoroutinesDumpTest\$testCreationStackTrace\$1.invokeSuspend(CoroutinesDumpTest.kt)").trimStackTrace() + assertTrue(result.startsWith(expected), "Actual:\n$result") } @Test @@ -186,4 +182,15 @@ class CoroutinesDumpTest : DebugTestBase() { (monitor as Object).notifyAll() } } + + private fun assertStartsWith(expected: String, actual: String) { + if (!actual.startsWith(expected)) { + println("----- Expected prefix") + println(expected) + println("----- Actual") + println(actual) + println("-----") + assertEquals(expected, actual) + } + } } diff --git a/kotlinx-coroutines-debug/test/DebugProbesTest.kt b/kotlinx-coroutines-debug/test/DebugProbesTest.kt index 4b39438138..f4d3e38e44 100644 --- a/kotlinx-coroutines-debug/test/DebugProbesTest.kt +++ b/kotlinx-coroutines-debug/test/DebugProbesTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines.debug @@ -39,25 +39,25 @@ class DebugProbesTest : DebugTestBase() { val deferred = createDeferred() val traces = listOf( "java.util.concurrent.ExecutionException\n" + - "\tat kotlinx.coroutines.debug.DebugProbesTest\$createDeferred\$1.invokeSuspend(DebugProbesTest.kt)\n" + + "\tat kotlinx.coroutines.debug.DebugProbesTest\$createDeferred\$1.invokeSuspend(DebugProbesTest.kt:16)\n" + "\t(Coroutine boundary)\n" + - "\tat kotlinx.coroutines.debug.DebugProbesTest.oneMoreNestedMethod(DebugProbesTest.kt)\n" + - "\tat kotlinx.coroutines.debug.DebugProbesTest.nestedMethod(DebugProbesTest.kt)\n" + - "\tat kotlinx.coroutines.debug.DebugProbesTest\$testAsyncWithProbes\$1\$1.invokeSuspend(DebugProbesTest.kt:62)\n" + + "\tat kotlinx.coroutines.debug.DebugProbesTest.oneMoreNestedMethod(DebugProbesTest.kt:71)\n" + + "\tat kotlinx.coroutines.debug.DebugProbesTest.nestedMethod(DebugProbesTest.kt:66)\n" + + "\tat kotlinx.coroutines.debug.DebugProbesTest\$testAsyncWithProbes\$1\$1.invokeSuspend(DebugProbesTest.kt:62)\n" + "\t(Coroutine creation stacktrace)\n" + - "\tat kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt)\n" + - "\tat kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt)\n" + - "\tat kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable\$default(Cancellable.kt)\n" + - "\tat kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt)\n" + - "\tat kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt)\n" + - "\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt)\n" + + "\tat kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt:116)\n" + + "\tat kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt:26)\n" + + "\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.startCoroutineImpl(Builders.common.kt:179)\n" + + "\tat kotlinx.coroutines.BuildersKt.startCoroutineImpl(Unknown Source)\n" + + "\tat kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt:145)\n" + + "\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:55)\n" + "\tat kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source)\n" + - "\tat kotlinx.coroutines.TestBase.runTest(TestBase.kt)\n" + - "\tat kotlinx.coroutines.TestBase.runTest\$default(TestBase.kt)\n" + - "\tat kotlinx.coroutines.debug.DebugProbesTest.testAsyncWithProbes(DebugProbesTest.kt)", + "\tat kotlinx.coroutines.TestBase.runTest(TestBase.kt:188)\n" + + "\tat kotlinx.coroutines.TestBase.runTest\$default(TestBase.kt:182)\n" + + "\tat kotlinx.coroutines.debug.DebugProbesTest.testAsyncWithProbes(DebugProbesTest.kt:39)", "Caused by: java.util.concurrent.ExecutionException\n" + - "\tat kotlinx.coroutines.debug.DebugProbesTest\$createDeferred\$1.invokeSuspend(DebugProbesTest.kt)\n" + - "\tat kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt)\n") + "\tat kotlinx.coroutines.debug.DebugProbesTest\$createDeferred\$1.invokeSuspend(DebugProbesTest.kt:16)\n" + + "\tat kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:32)\n") nestedMethod(deferred, traces) deferred.join() } diff --git a/kotlinx-coroutines-test/common/src/TestScope.kt b/kotlinx-coroutines-test/common/src/TestScope.kt index 60585a1d50..a20f720847 100644 --- a/kotlinx-coroutines-test/common/src/TestScope.kt +++ b/kotlinx-coroutines-test/common/src/TestScope.kt @@ -4,6 +4,7 @@ package kotlinx.coroutines.test +import kotlinx.atomicfu.* import kotlinx.coroutines.* import kotlinx.coroutines.internal.* import kotlin.coroutines.* @@ -165,18 +166,18 @@ internal class TestScopeImpl(context: CoroutineContext) : override val testScheduler get() = context[TestCoroutineScheduler]!! - private var entered = false - private var finished = false + private val entered = atomic(false) + private val finished = atomic(false) private val uncaughtExceptions = mutableListOf() private val lock = SynchronizedObject() /** Called upon entry to [runTest]. Will throw if called more than once. */ fun enter() { val exceptions = synchronized(lock) { - if (entered) + if (entered.value) throw IllegalStateException("Only a single call to `runTest` can be performed during one test.") - entered = true - check(!finished) + entered.value = true + check(!finished.value) uncaughtExceptions } if (exceptions.isNotEmpty()) { @@ -190,9 +191,9 @@ internal class TestScopeImpl(context: CoroutineContext) : /** Called at the end of the test. May only be called once. */ fun leave(): List { val exceptions = synchronized(lock) { - if(!entered || finished) + if(!entered.value || finished.value) throw IllegalStateException("An internal error. Please report to the Kotlinx Coroutines issue tracker") - finished = true + finished.value = true uncaughtExceptions } val activeJobs = children.filter { it.isActive }.toList() // only non-empty if used with `runBlockingTest` @@ -215,11 +216,11 @@ internal class TestScopeImpl(context: CoroutineContext) : /** Stores an exception to report after [runTest], or rethrows it if not inside [runTest]. */ fun reportException(throwable: Throwable) { synchronized(lock) { - if (finished) { + if (finished.value) { throw throwable } else { uncaughtExceptions.add(throwable) - if (!entered) + if (!entered.value) throw UncaughtExceptionsBeforeTest().apply { addSuppressed(throwable) } } } @@ -229,7 +230,7 @@ internal class TestScopeImpl(context: CoroutineContext) : fun tryGetCompletionCause(): Throwable? = completionCause override fun toString(): String = - "TestScope[" + (if (finished) "test ended" else if (entered) "test started" else "test not started") + "]" + "TestScope[" + (if (finished.value) "test ended" else if (entered.value) "test started" else "test not started") + "]" } /** Use the knowledge that any [TestScope] that we receive is necessarily a [TestScopeImpl]. */ diff --git a/kotlinx-coroutines-test/common/test/RunTestTest.kt b/kotlinx-coroutines-test/common/test/RunTestTest.kt index 3b6272c062..8f1ecab757 100644 --- a/kotlinx-coroutines-test/common/test/RunTestTest.kt +++ b/kotlinx-coroutines-test/common/test/RunTestTest.kt @@ -10,6 +10,7 @@ import kotlinx.coroutines.flow.* import kotlin.coroutines.* import kotlin.test.* +@NoNative class RunTestTest { /** Tests that [withContext] that sends work to other threads works in [runTest]. */ diff --git a/kotlinx-coroutines-test/native/test/FailingTests.kt b/kotlinx-coroutines-test/native/test/FailingTests.kt index 9fb77ce7c8..f4a60538fb 100644 --- a/kotlinx-coroutines-test/native/test/FailingTests.kt +++ b/kotlinx-coroutines-test/native/test/FailingTests.kt @@ -9,6 +9,7 @@ import kotlin.test.* /** These are tests that we want to fail. They are here so that, when the issue is fixed, their failure indicates that * everything is better now. */ +@NoNative class FailingTests { @Test fun testRunTestLoopShutdownOnTimeout() = testResultMap({ fn -> @@ -22,4 +23,4 @@ class FailingTests { } } -} \ No newline at end of file +} diff --git a/reactive/kotlinx-coroutines-rx2/test/ObservableSingleTest.kt b/reactive/kotlinx-coroutines-rx2/test/ObservableSingleTest.kt index e246407a17..acc17a2b44 100644 --- a/reactive/kotlinx-coroutines-rx2/test/ObservableSingleTest.kt +++ b/reactive/kotlinx-coroutines-rx2/test/ObservableSingleTest.kt @@ -137,7 +137,7 @@ class ObservableSingleTest : TestBase() { @Test fun testAwaitFirstOrElseWithValues() { val observable = rxObservable { - send(Observable.just("O", "#").awaitFirstOrElse { "!" } + "K") + send(Observable.just("O", "#").awaitFirstOrElse { null } + "K") } checkSingleValue(observable) {