feat: transcribe (#89)

* init

* feat: update, array buffer support

* tweaks

* tweaks

* tweaks

Author: grabbou

apps/example-apple/src/schema-demos.ts

@@ -1,14 +1,5 @@
 import { createAppleProvider } from '@react-native-ai/apple'
-import NativeAppleEmbeddings from '@react-native-ai/apple/src/NativeAppleEmbeddings'
-import {
-  cosineSimilarity,
-  embed,
-  embedMany,
-  generateObject,
-  generateText,
-  streamText,
-  tool,
-} from 'ai'
+import { experimental_transcribe, generateObject, streamText, tool } from 'ai'
 import { z } from 'zod'
 
 const getWeather = tool({
@@ -125,6 +116,18 @@ export async function basicArrayDemo() {
   return response.object
 }
 
+export async function basicTranscribeDemo() {
+  const file = await fetch(
+    'https://www.voiptroubleshooter.com/open_speech/american/OSR_us_000_0010_8k.wav'
+  )
+  const audio = await file.arrayBuffer()
+  const response = await experimental_transcribe({
+    model: apple.transcriptionModel(),
+    audio,
+  })
+  return response.text
+}
+
 export const schemaDemos = {
   basicString: { name: 'String', func: basicStringDemo },
   basicStringStreaming: {
@@ -136,6 +139,7 @@ export const schemaDemos = {
   basicBoolean: { name: 'Boolean', func: basicBooleanDemo },
   basicObject: { name: 'Object', func: basicObjectDemo },
   basicArray: { name: 'Array', func: basicArrayDemo },
+  basicTranscribe: { name: 'Transcribe', func: basicTranscribeDemo },
 }
 
 export type DemoKey = keyof typeof schemaDemos

apps/example-apple/src/screens/SampleRuns.tsx

@@ -119,4 +119,4 @@ const styles = StyleSheet.create({
   spinner: {
     marginRight: 8,
   },
-})
+})

packages/apple-llm/ios/embeddings/AppleEmbeddingsImpl.swift

@@ -75,126 +75,128 @@ public class AppleEmbeddingsImpl: NSObject {
     case unsupportedLanguage(String)
   }
 
+  @available(iOS 16, *)
   private func convertToNLLanguage(_ languageString: String) throws -> NLLanguage {
-    switch languageString.lowercased() {
-    case "english", "en":
+    let locale = Locale(identifier: languageString)
+    guard let languageCode = locale.language.languageCode else {
+      throw LanguageConversionError.unsupportedLanguage("Cannot extract language code from locale: \(locale)")
+    }
+    
+    switch languageCode {
+    case "en":
       return .english
-    case "french", "fr":
+    case "fr":
       return .french
-    case "spanish", "es":
+    case "es":
       return .spanish
-    case "german", "de":
+    case "de":
       return .german
-    case "italian", "it":
+    case "it":
       return .italian
-    case "portuguese", "pt":
+    case "pt":
       return .portuguese
-    case "russian", "ru":
+    case "ru":
       return .russian
-    case "turkish", "tr":
+    case "tr":
       return .turkish
-    case "chinese", "zh":
+    case "zh":
       return .simplifiedChinese
-    case "arabic", "ar":
+    case "ar":
       return .arabic
-    case "czech", "cs":
+    case "cs":
       return .czech
-    case "dutch", "nl":
+    case "nl":
       return .dutch
-    case "finnish", "fi":
+    case "fi":
       return .finnish
-    case "hebrew", "he":
+    case "he":
       return .hebrew
-    case "hindi", "hi":
+    case "hi":
       return .hindi
-    case "hungarian", "hu":
+    case "hu":
       return .hungarian
-    case "icelandic", "is":
+    case "is":
       return .icelandic
-    case "indonesian", "id":
+    case "id":
       return .indonesian
-    case "japanese", "ja":
+    case "ja":
       return .japanese
-    case "korean", "ko":
+    case "ko":
       return .korean
-    case "malay", "ms":
+    case "ms":
       return .malay
-    case "norwegian", "no":
+    case "no":
       return .norwegian
-    case "polish", "pl":
+    case "pl":
       return .polish
-    case "romanian", "ro":
+    case "ro":
       return .romanian
-    case "slovak", "sk":
+    case "sk":
       return .slovak
-    case "swedish", "sv":
+    case "sv":
       return .swedish
-    case "thai", "th":
+    case "th":
       return .thai
-    case "ukrainian", "uk":
+    case "uk":
       return .ukrainian
-    case "amharic", "am":
+    case "am":
       return .amharic
-    case "armenian", "hy":
+    case "hy":
       return .armenian
-    case "bengali", "bn":
+    case "bn":
       return .bengali
-    case "bulgarian", "bg":
+    case "bg":
       return .bulgarian
-    case "burmese", "my":
+    case "my":
       return .burmese
-    case "catalan", "ca":
+    case "ca":
       return .catalan
-    case "cherokee", "chr":
+    case "chr":
       return .cherokee
-    case "croatian", "hr":
+    case "hr":
       return .croatian
-    case "danish", "da":
+    case "da":
       return .danish
-    case "georgian", "ka":
+    case "ka":
       return .georgian
-    case "greek", "el":
+    case "el":
       return .greek
-    case "gujarati", "gu":
+    case "gu":
       return .gujarati
-    case "kannada", "kn":
+    case "kn":
       return .kannada
-    case "khmer", "km":
+    case "km":
       return .khmer
-    case "lao", "lo":
+    case "lo":
       return .lao
-    case "malayalam", "ml":
+    case "ml":
       return .malayalam
-    case "marathi", "mr":
+    case "mr":
       return .marathi
-    case "mongolian", "mn":
+    case "mn":
       return .mongolian
-    case "oriya", "or":
+    case "or":
       return .oriya
-    case "persian", "fa":
+    case "fa":
       return .persian
-    case "punjabi", "pa":
+    case "pa":
       return .punjabi
-    case "sinhalese", "si":
+    case "si":
       return .sinhalese
-    case "tamil", "ta":
+    case "ta":
       return .tamil
-    case "telugu", "te":
+    case "te":
       return .telugu
-    case "tibetan", "bo":
+    case "bo":
       return .tibetan
-    case "urdu", "ur":
+    case "ur":
       return .urdu
-    case "vietnamese", "vi":
+    case "vi":
       return .vietnamese
-    case "kazakh", "kk":
-      if #available(iOS 16.0, *) {
-        return .kazakh
-      } else {
-        throw LanguageConversionError.unsupportedLanguage("\(languageString) (requires iOS 16+)")
-      }
+    case "kk":
+      return .kazakh
     default:
-      throw LanguageConversionError.unsupportedLanguage(languageString)
+      throw LanguageConversionError.unsupportedLanguage("\(languageCode)")
     }
   }
 

packages/apple-llm/ios/speech/AppleSpeech.mm

@@ -0,0 +1,122 @@
+//
+//  AppleSpeech.mm
+//  AppleLLM
+//
+//  Created by Mike Grabowski on 01/08/2025.
+//
+
+#if __has_include("AppleLLM/AppleLLM-Swift.h")
+#import "AppleLLM/AppleLLM-Swift.h"
+#else
+#import "AppleLLM-Swift.h"
+#endif
+
+#import <React/RCTCallInvokerModule.h>
+#import <React/RCTCallInvoker.h>
+#import <ReactCommon/RCTTurboModule.h>
+
+#import <jsi/jsi.h>
+
+#import <NativeAppleLLM/NativeAppleLLM.h>
+
+@interface AppleSpeech : NativeAppleSpeechSpecBase <NativeAppleSpeechSpec, RCTCallInvokerModule>
+@property (strong, nonatomic) AppleSpeechImpl *speech;
+@end
+
+using namespace facebook;
+using namespace JS::NativeAppleLLM;
+
+@implementation AppleSpeech
+
+@synthesize callInvoker;
+
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    _speech = [AppleSpeechImpl new];
+  }
+  return self;
+}
+
++ (NSString *)moduleName {
+  return @"NativeAppleSpeech";
+}
+
+- (void)installTranscribeFunc:(std::shared_ptr<facebook::react::CallInvoker>)jsInvoker {
+  AppleSpeechImpl *speechModule = _speech;
+  
+  auto runOnJS = [jsInvoker](std::function<void()>&& f) {
+    jsInvoker->invokeAsync(std::move(f));
+  };
+  
+  jsInvoker->invokeAsync([speechModule, runOnJS](jsi::Runtime& rt) {
+    @try {
+      auto global = rt.global();
+      
+      auto transcribeFunc = jsi::Function::createFromHostFunction(
+        rt,
+        jsi::PropNameID::forAscii(rt, "transcribe"),
+        2,
+        [speechModule, runOnJS](jsi::Runtime& rt, const jsi::Value& thisVal, const jsi::Value* args, size_t count) -> jsi::Value {
+          auto arrayBuffer = args[0].asObject(rt);
+          if (!arrayBuffer.isArrayBuffer(rt)) {
+            throw jsi::JSError(rt, "First argument must be an ArrayBuffer");
+          }
+          
+          auto buffer = arrayBuffer.getArrayBuffer(rt);
+          NSData *audioData = [NSData dataWithBytes:buffer.data(rt) length:buffer.size(rt)];
+          
+          auto languageStr = args[1].asString(rt).utf8(rt);
+          NSString *language = [NSString stringWithUTF8String:languageStr.c_str()];
+          
+          auto Promise = rt.global().getPropertyAsFunction(rt, "Promise");
+          
+          return Promise.callAsConstructor(rt, jsi::Function::createFromHostFunction(
+            rt,
+            jsi::PropNameID::forAscii(rt, "executor"),
+            2,
+            [speechModule, audioData, language, runOnJS](jsi::Runtime& rt, const jsi::Value& thisVal, const jsi::Value* args, size_t count) -> jsi::Value {
+              auto resolve = std::make_shared<jsi::Function>(args[0].asObject(rt).asFunction(rt));
+              auto reject = std::make_shared<jsi::Function>(args[1].asObject(rt).asFunction(rt));
+              
+              [speechModule transcribe:audioData
+                              language:language
+                               resolve:^(id result) {
+                                 runOnJS([resolve, result, &rt]() {
+                                   auto jsResult = react::TurboModuleConvertUtils::convertObjCObjectToJSIValue(rt, result);
+                                   resolve->call(rt, jsResult);
+                                 });
+                              } reject:^(NSString *code, NSString *message, NSError *error) {
+                                 runOnJS([reject, message, &rt]() {
+                                   auto jsError = jsi::String::createFromUtf8(rt, [message UTF8String]);
+                                   reject->call(rt, jsError);
+                                 });
+                              }];
+              
+              return jsi::Value::undefined();
+            }
+          ));
+        }
+      );
+      
+      global.setProperty(rt, "__apple__llm__transcribe__", transcribeFunc);
+    } @catch (NSException *exception) {
+      throw jsi::JSError(rt, [[NSString stringWithFormat:@"Failed to install transcribe handler: %@", exception.reason] UTF8String]);
+    }
+  });
+}
+
+- (std::shared_ptr<react::TurboModule>)getTurboModule:(const react::ObjCTurboModule::InitParams &)params {
+  [self installTranscribeFunc:params.jsInvoker];
+  return std::make_shared<react::NativeAppleSpeechSpecJSI>(params);
+}
+
+- (nonnull NSNumber *)isAvailable:(nonnull NSString *)language { 
+  return @([_speech isAvailable:language]);
+}
+
+- (void)prepare:(nonnull NSString *)language resolve:(nonnull RCTPromiseResolveBlock)resolve reject:(nonnull RCTPromiseRejectBlock)reject { 
+  [_speech prepare:language resolve:resolve reject:reject];
+}
+
+@end