Android Streaming Demo

.gitignore

@@ -34,7 +34,7 @@ build-sanitize-thread/
 /quantize
 /server
 /lsp
-
+/models
 arm_neon.h
 sync.sh
 libwhisper.a

examples/whisper.android/.gitignore

@@ -13,3 +13,6 @@
 .externalNativeBuild
 .cxx
 local.properties
+/app/src/main/assets/
+/app/src/main/assets/models/
+

examples/whisper.android/app/.gitignore

@@ -1 +1 @@
-/build
+/build

examples/whisper.android/app/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'com.android.application'
     id 'org.jetbrains.kotlin.android'
+
 }
 
 android {
@@ -9,7 +10,7 @@ android {
 
     defaultConfig {
         applicationId "com.whispercppdemo"
-        minSdk 26
+        minSdk 31
         targetSdk 34
         versionCode 1
         versionName "1.0"
@@ -29,31 +30,44 @@ android {
         }
     }
     compileOptions {
-        sourceCompatibility JavaVersion.VERSION_17
-        targetCompatibility JavaVersion.VERSION_17
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
     }
     kotlinOptions {
-        jvmTarget = '17'
+        jvmTarget = '1.8'
     }
     buildFeatures {
         compose true
     }
     composeOptions {
-        kotlinCompilerExtensionVersion '1.5.0'
+        kotlinCompilerExtensionVersion '1.5.2'
     }
+    ndkVersion = "25.2.9519653"
 }
 
 dependencies {
-    implementation project(':lib')
-    implementation 'androidx.activity:activity-compose:1.7.2'
-    implementation 'androidx.compose.material:material-icons-core:1.5.0'
-    implementation 'androidx.compose.material3:material3:1.1.1'
-    implementation "androidx.compose.ui:ui:1.5.0"
-    implementation "androidx.compose.ui:ui-tooling-preview:1.5.0"
-    implementation 'androidx.lifecycle:lifecycle-viewmodel-compose:2.6.1'
-    implementation "com.google.accompanist:accompanist-permissions:0.28.0"
-    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.2'
 
+    implementation(project(":lib"))
+    implementation("androidx.compose.ui:ui:1.5.2")
+    implementation("androidx.compose.material:material:1.5.2")
+    implementation("androidx.activity:activity-compose:1.7.2")
+    implementation("androidx.compose.material:material-icons-core:1.5.0")
+    implementation("androidx.compose.material3:material3:1.1.1")
+    implementation("androidx.compose.ui:ui:1.5.2")
+    implementation("androidx.compose.ui:ui-tooling-preview:1.5.2")
+    implementation("androidx.compose.runtime:runtime-livedata:1.5.2")
+    implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.6.1")
+    implementation("com.google.accompanist:accompanist-permissions:0.28.0")
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.2")
+    implementation("androidx.lifecycle:lifecycle-viewmodel-ktx:2.7.0")
+    implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.3.1")
+
+
+
+
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.2")
+    implementation("androidx.lifecycle:lifecycle-viewmodel-ktx:2.7.0")
+    implementation ("androidx.lifecycle:lifecycle-runtime-ktx:2.3.1")
     testImplementation 'junit:junit:4.13.2'
     androidTestImplementation 'androidx.test.ext:junit:1.1.5'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'

examples/whisper.android/app/src/main/java/com/whispercppdemo/recorder/Recorder.kt

@@ -11,78 +11,165 @@ import kotlinx.coroutines.withContext
 import java.io.File
 import java.util.concurrent.Executors
 import java.util.concurrent.atomic.AtomicBoolean
+import android.util.Log
 
-class Recorder {
+import kotlinx.coroutines.runBlocking
+
+private const val TAG = "Recorder"
+
+
+
+class Recorder() {
     private val scope: CoroutineScope = CoroutineScope(
         Executors.newSingleThreadExecutor().asCoroutineDispatcher()
     )
+
     private var recorder: AudioRecordThread? = null
+    private var audioStream: AudioStreamThread? = null
+
 
-    suspend fun startRecording(outputFile: File, onError: (Exception) -> Unit) = withContext(scope.coroutineContext) {
-        recorder = AudioRecordThread(outputFile, onError)
-        recorder?.start()
+    suspend fun startRecording(outputFile: File, onError: (Exception) -> Unit) =
+        withContext(scope.coroutineContext) {
+            recorder = AudioRecordThread(outputFile, onError)
+            recorder?.start()
+        }
+
+    fun startStreaming(onDataReceived: AudioDataReceivedListener, onError: (Exception) -> Unit) {
+        if (audioStream == null) {
+            audioStream = AudioStreamThread(onDataReceived, onError)
+            audioStream?.start()
+        } else {
+            Log.i(TAG, "AudioStreamThread is already running")
+        }
     }
 
-    suspend fun stopRecording() = withContext(scope.coroutineContext) {
+
+    fun stopRecording() {
         recorder?.stopRecording()
-        @Suppress("BlockingMethodInNonBlockingContext")
-        recorder?.join()
-        recorder = null
+        audioStream?.stopRecording()
+        runBlocking {
+            audioStream?.join()
+            audioStream = null
+            recorder?.join()
+            recorder = null
+        }
     }
-}
 
-private class AudioRecordThread(
-    private val outputFile: File,
-    private val onError: (Exception) -> Unit
-) :
-    Thread("AudioRecorder") {
-    private var quit = AtomicBoolean(false)
 
-    @SuppressLint("MissingPermission")
-    override fun run() {
-        try {
+    private class AudioRecordThread(
+        private val outputFile: File,
+        private val onError: (Exception) -> Unit
+    ) :
+        Thread("AudioRecorder") {
+        private var quit = AtomicBoolean(false)
+
+        @SuppressLint("MissingPermission")
+        override fun run() {
+            try {
+                val bufferSize = AudioRecord.getMinBufferSize(
+                    16000,
+                    AudioFormat.CHANNEL_IN_MONO,
+                    AudioFormat.ENCODING_PCM_16BIT
+                ) * 4
+                val buffer = ShortArray(bufferSize / 2)
+
+                val audioRecord = AudioRecord(
+                    MediaRecorder.AudioSource.MIC,
+                    16000,
+                    AudioFormat.CHANNEL_IN_MONO,
+                    AudioFormat.ENCODING_PCM_16BIT,
+                    bufferSize
+                )
+
+                try {
+                    audioRecord.startRecording()
+
+                    val allData = mutableListOf<Short>()
+
+                    while (!quit.get()) {
+                        val read = audioRecord.read(buffer, 0, buffer.size)
+                        if (read > 0) {
+                            for (i in 0 until read) {
+                                allData.add(buffer[i])
+                            }
+                        } else {
+                            throw java.lang.RuntimeException("audioRecord.read returned $read")
+                        }
+                    }
+
+                    audioRecord.stop()
+                    encodeWaveFile(
+                        outputFile,
+                        allData.toShortArray()
+                    )
+                } finally {
+                    audioRecord.release()
+                }
+            } catch (e: Exception) {
+                onError(e)
+            }
+        }
+
+        fun stopRecording() {
+            quit.set(true)
+        }
+
+
+    }
+
+    interface AudioDataReceivedListener {
+        fun onAudioDataReceived(data: FloatArray)
+    }
+    private class AudioStreamThread(
+        private val onDataReceived: AudioDataReceivedListener,
+        private val onError: (Exception) -> Unit
+    ) : Thread("AudioStreamer") {
+        private val quit = AtomicBoolean(false)
+
+        @SuppressLint("MissingPermission")
+        override fun run() {
             val bufferSize = AudioRecord.getMinBufferSize(
                 16000,
                 AudioFormat.CHANNEL_IN_MONO,
-                AudioFormat.ENCODING_PCM_16BIT
-            ) * 4
-            val buffer = ShortArray(bufferSize / 2)
-
+                AudioFormat.ENCODING_PCM_FLOAT) * 4
+            val floatBuffer = FloatArray(bufferSize / 2)
             val audioRecord = AudioRecord(
                 MediaRecorder.AudioSource.MIC,
                 16000,
                 AudioFormat.CHANNEL_IN_MONO,
-                AudioFormat.ENCODING_PCM_16BIT,
-                bufferSize
-            )
+                AudioFormat.ENCODING_PCM_FLOAT,
+                bufferSize)
+
+            if (audioRecord.state != AudioRecord.STATE_INITIALIZED) {
+                Log.e(TAG, "AudioRecord initialization failed")
+                return
+            }
 
             try {
                 audioRecord.startRecording()
+                while (!quit.get()) {
 
-                val allData = mutableListOf<Short>()
 
-                while (!quit.get()) {
-                    val read = audioRecord.read(buffer, 0, buffer.size)
-                    if (read > 0) {
-                        for (i in 0 until read) {
-                            allData.add(buffer[i])
-                        }
-                    } else {
-                        throw java.lang.RuntimeException("audioRecord.read returned $read")
+                    val readResult = audioRecord.read(floatBuffer, 0, floatBuffer.size, AudioRecord.READ_BLOCKING)
+                    Log.i(TAG, "readResult: $readResult")
+                    if (readResult > 0) {
+                        Log.i(TAG, "READING FROM THE floatBuffer")
+
+                        onDataReceived.onAudioDataReceived(floatBuffer.copyOf(readResult))
+                    } else if (readResult < 0) {
+                        throw RuntimeException("AudioRecord.read error: $readResult")
                     }
                 }
-
-                audioRecord.stop()
-                encodeWaveFile(outputFile, allData.toShortArray())
+            } catch (e: Exception) {
+                onError(e)
             } finally {
+                audioRecord.stop()
                 audioRecord.release()
             }
-        } catch (e: Exception) {
-            onError(e)
         }
-    }
 
-    fun stopRecording() {
-        quit.set(true)
+        fun stopRecording() {
+            quit.set(true)
+        }
     }
-}
+}