refactor: improved combined Locator for MediaOverlay playback position sent back to Flutter

Author: ddfreiling

flutter_readium/ios/flutter_readium/Sources/flutter_readium/FlutterReadiumPlugin.swift

@@ -7,6 +7,7 @@ import ReadiumShared
 
 private let TAG = "ReadiumReaderPlugin"
 
+internal var currentPublicationUrlStr: String?
 internal var currentPublication: Publication?
 internal var currentReaderView: ReadiumReaderView?
 
@@ -93,6 +94,7 @@ public class FlutterReadiumPlugin: NSObject, FlutterPlugin, ReadiumShared.Warnin
           }
           let pub: Publication = try await self.loadPublication(fromUrlStr: pubUrlStr).get()
           currentPublication = pub
+          currentPublicationUrlStr = pubUrlStr
 
           let jsonManifest = pub.jsonManifest
           await MainActor.run {
@@ -333,10 +335,11 @@ public class FlutterReadiumPlugin: NSObject, FlutterPlugin, ReadiumShared.Warnin
       }
     case "audioEnable":
       guard let args = call.arguments as? [Any?],
-            let publication = currentPublication else {
+            let publication = currentPublication,
+            let pubUrlStr = currentPublicationUrlStr else {
         return result(FlutterError.init(
           code: "audioEnable",
-          message: "Invalid parameters to audioEnable: \(call.arguments.debugDescription)",
+          message: "No publication open or Invalid parameters to audioEnable: \(call.arguments.debugDescription)",
           details: nil))
       }
       Task.detached(priority: .high) {
@@ -349,13 +352,23 @@ public class FlutterReadiumPlugin: NSObject, FlutterPlugin, ReadiumShared.Warnin
         }
 
         if (publication.containsMediaOverlays) {
-          // TODO: May have to re-load publication from URL and pass the new copy of Publication here.
-          self.timebasedNavigator = await FlutterMediaOverlayNavigator(publication: publication, preferences: prefs, initialLocator: locator)
+          do {
+            // MediaOverlayNavigator will modify the Publication readingOrder, so we first load a modifiable copy.
+            let modifiablePublicationCopy = try await self.loadPublication(fromUrlStr: pubUrlStr).get()
+            await MainActor.run { [locator] in
+              self.timebasedNavigator = FlutterMediaOverlayNavigator(publication: modifiablePublicationCopy, preferences: prefs, initialLocator: locator)
+            }
+          } catch (let err) {
+            return result(FlutterError.init(
+              code: "Error",
+              message: "Failed to reload a modifiable publication copy from: \(pubUrlStr)",
+              details: err))
+          }
         } else {
           if (!publication.conforms(to: Publication.Profile.audiobook)) {
             return result(FlutterError.init(
               code: "ArgumentError",
-              message: "Publication does not contain MediaOverlays or conformTo AudioBook: \(call.arguments.debugDescription)",
+              message: "Publication does not contain MediaOverlays or conforms to AudioBook profile. Args: \(call.arguments.debugDescription)",
               details: nil))
           }
           self.timebasedNavigator = await FlutterAudioNavigator(publication: publication, preferences: prefs, initialLocator: locator)
@@ -364,7 +377,9 @@ public class FlutterReadiumPlugin: NSObject, FlutterPlugin, ReadiumShared.Warnin
         self.timebasedNavigator?.listener = self
         await self.timebasedNavigator?.initNavigator()
 
-        result(nil)
+        await MainActor.run {
+          result(nil)
+        }
       }
     case "audioSetPreferences":
       Task.detached(priority: .high) {
@@ -495,6 +510,7 @@ extension FlutterReadiumPlugin {
       self.timebasedNavigator = nil
       currentPublication?.close()
       currentPublication = nil
+      currentPublicationUrlStr = nil
     }
   }
 }

flutter_readium/ios/flutter_readium/Sources/flutter_readium/model/FlutterMediaOverlay.swift

@@ -160,6 +160,16 @@ final class FlutterMediaOverlayItem: NSObject {
     )
   }
 
+  func toCombinedLocator(fromPlaybackLocator audioLocator: Locator) -> Locator? {
+    guard var textLocator = self.asTextLocator else { return nil }
+    textLocator.mediaType = .mpegAudio
+    textLocator.locations.progression = audioLocator.locations.progression
+    textLocator.locations.totalProgression = audioLocator.locations.totalProgression
+    textLocator.locations.position = audioLocator.locations.position
+    textLocator.locations.fragments = textLocator.locations.fragments + audioLocator.locations.fragments
+    return textLocator
+  }
+  
   // MARK: JSON
   static func fromJson(_ json: [String: Any], atPosition position: Int) -> FlutterMediaOverlayItem? {
     guard

flutter_readium/ios/flutter_readium/Sources/flutter_readium/navigator/FlutterAudioNavigator.swift

@@ -180,7 +180,8 @@ public class FlutterAudioNavigator: FlutterTimebasedNavigator, AudioNavigatorDel
   }
 
   public func navigator(_ navigator: any ReadiumNavigator.Navigator, presentError error: ReadiumNavigator.NavigatorError) {
-    print(TAG, "presentError: \(error)")
+    debugPrint(TAG, "presentError: \(error)")
+    // TODO: Only relevant when supporting LCP, error can only be copyForbidden.
   }
 
   public func navigator(_ navigator: any ReadiumNavigator.Navigator, didFailToLoadResourceAt href: ReadiumShared.RelativeURL, withError error: ReadiumShared.ReadError) {
@@ -189,6 +190,7 @@ public class FlutterAudioNavigator: FlutterTimebasedNavigator, AudioNavigatorDel
 
   // MARK: AudioNavigator specific API
 
+  @MainActor
   func setAudioPreferences(_ preferences: FlutterAudioPreferences) {
     self._preferences = preferences
     self._audioNavigator?.submitPreferences(AudioPreferences(fromFlutterPrefs: preferences))
@@ -202,13 +204,15 @@ public class FlutterAudioNavigator: FlutterTimebasedNavigator, AudioNavigatorDel
     self._audioNavigator?.canGoForward ?? false
   }
 
+  @MainActor
   public func skipForward() async -> Bool {
     if _audioNavigator?.canGoForward != true {
       return false
     }
     return await _audioNavigator?.goForward() ?? false
   }
 
+  @MainActor
   public func skipBackward() async -> Bool {
     if _audioNavigator?.canGoBackward != true {
       return false
@@ -238,7 +242,7 @@ public class FlutterAudioNavigator: FlutterTimebasedNavigator, AudioNavigatorDel
       self._lastTimebasedPlayerState = state
       self.listener?.timebasedNavigator(self, didChangeState: state)
     } else {
-      print(TAG, "Skipped state submission - duplicate")
+      debugPrint(TAG, "Skipped state submission - duplicate")
     }
   }
 }

flutter_readium/ios/flutter_readium/Sources/flutter_readium/navigator/FlutterMediaOverlayNavigator.swift

@@ -17,8 +17,9 @@ public class FlutterMediaOverlayNavigator : FlutterAudioNavigator
 
   public override init(publication: Publication, preferences: FlutterAudioPreferences, initialLocator: Locator?) {
     super.init(publication: publication, preferences: preferences, initialLocator: initialLocator)
+    
     // Map the initial Text-based locator to Audio-based MediaOverlay Locator.
-    self._initialLocator = self.mapTextLocatorToMediaOverlayLocator(initialLocator)
+    self._initialLocator = self.mapTextLocatorToMediaOverlayAudioLocator(initialLocator)
   }
 
   public override func initNavigator() async -> Void {
@@ -50,29 +51,26 @@ public class FlutterMediaOverlayNavigator : FlutterAudioNavigator
     audioPubManifest.readingOrder = audioReadingOrder
     audioPubManifest.metadata.conformsTo = [Publication.Profile.audiobook]
 
-    // TODO: This modifies the existing Publication reference !!!
-    // Instead we may need to re-load the Publication from same URL, to get a separate reference.
-    var newPub = publication
-    newPub.manifest = audioPubManifest
+    // Note: This modifies the Publication reference !!!
+    // For now caller must re-load the Publication from same URL, to get a separate reference.
+    publication.manifest = audioPubManifest
 
     debugPrint(OTAG, "New audio readingOrder found: \(audioReadingOrder)")
     // Save the media-overlays for later position matching.
     self.mediaOverlays = mediaOverlays
-    // Assign the publication, it should now conform to AudioBook.
-    self._publication = newPub
 
     await super.initNavigator()
   }
 
   override public func play(fromLocator: Locator?) async {
     // Map the initial Text-based locator to Audio-based MediaOverlay Locator.
-    let audioFromLocator = mapTextLocatorToMediaOverlayLocator(fromLocator)
+    let audioFromLocator = mapTextLocatorToMediaOverlayAudioLocator(fromLocator)
     await super.play(fromLocator: audioFromLocator)
   }
 
   override public func seek(toLocator: Locator) async -> Bool {
     guard let navigator = _audioNavigator,
-          let audioLocator = mapTextLocatorToMediaOverlayLocator(toLocator) else {
+          let audioLocator = mapTextLocatorToMediaOverlayAudioLocator(toLocator) else {
       return false
     }
     // Found a matching Audio Locator from given Text-based Locator.
@@ -88,28 +86,29 @@ public class FlutterMediaOverlayNavigator : FlutterAudioNavigator
     if let timeOffsetStr = location.locations.fragments.first(where: { $0.starts(with: "t=") })?.dropFirst(2),
        let timeOffset = Double(timeOffsetStr),
        let mediaOverlay = mediaOverlays.first(where: { $0.itemInRangeOfTime(timeOffset, inHref:  location.href.string) }),
-       var textLocator = mediaOverlay.asTextLocator {
-      if (!mediaOverlay.isEqual(lastMediaOverlayItem)) {
-        // Matched a new MediaOverlayItem -> sync reader with its textLocator.
-        lastMediaOverlayItem = mediaOverlay
-        textLocator.locations.progression = location.locations.progression
-        textLocator.locations.position = location.locations.position
-        
-        // TextLocator matching the audio position is created and should be sent back.
-        self.listener?.timebasedNavigator(self, reachedLocator: textLocator, readingOrderLink: nil)
-        self.listener?.timebasedNavigator(self, requestsHighlightAt: textLocator, withWordLocator: nil)
-      }
+       let combinedLocator = mediaOverlay.toCombinedLocator(fromPlaybackLocator: location) {
+
+      // Combined Text/Audio Locator matching the audio position is created and should be sent back.
+      self.listener?.timebasedNavigator(self, reachedLocator: combinedLocator, readingOrderLink: nil)
+      self.listener?.timebasedNavigator(self, requestsHighlightAt: combinedLocator, withWordLocator: nil)
     } else {
       debugPrint(OTAG, "Did not find MediaOverlay matching audio Locator: \(location)")
     }
   }
 
-  internal func mapTextLocatorToMediaOverlayLocator(_ textLocator: Locator?) -> Locator? {
+  internal func mapTextLocatorToMediaOverlayAudioLocator(_ textLocator: Locator?) -> Locator? {
     guard let textLocator = textLocator,
           let matchingItem = self.mediaOverlays.firstMap({ $0.itemFromLocator(textLocator)}),
-          let audioLocator = matchingItem.asAudioLocator else {
+          var audioLocator = matchingItem.asAudioLocator else {
       return nil
     }
+    // If the input Text Locator, is a combined locator with a time fragment
+    // we use this, as it can be more precise than the MediaOverlayItem fragment.
+    if let textLocatorTime = textLocator.locations.time,
+       let textLocatorTimeBegin = textLocatorTime.begin {
+      debugPrint(OTAG, "TextLocator had more precise time offset: \(textLocatorTimeBegin)")
+      audioLocator.locations.fragments = ["t=\(textLocatorTimeBegin)"]
+    }
     return audioLocator
   }
 }

flutter_readium/ios/flutter_readium/Sources/flutter_readium/navigator/FlutterTTSNavigator.swift

@@ -70,7 +70,7 @@ public class FlutterTTSNavigator: FlutterTimebasedNavigator, PublicationSpeechSy
         guard let self = self, let locator = locator else {
           return
         }
-        print(TAG, "tts send audio-locator")
+        debugPrint(TAG, "tts send audio-locator")
         let chapterNo = publication.readingOrder.firstIndexWithHREF(locator.href)
         let link = self.publication.readingOrder.firstWithHREF(locator.href)
 
@@ -89,7 +89,7 @@ public class FlutterTTSNavigator: FlutterTimebasedNavigator, PublicationSpeechSy
           return
         }
 
-        print(TAG, "sync reader to locator")
+        debugPrint(TAG, "sync reader to locator")
         let link = self.publication.readingOrder.firstWithHREF(locator.href)
         listener?.timebasedNavigator(self, reachedLocator: locator, readingOrderLink: link)
       }
@@ -150,7 +150,6 @@ public class FlutterTTSNavigator: FlutterTimebasedNavigator, PublicationSpeechSy
     return true
   }
 
-  @MainActor
   public func seek(toOffset: Double) async -> Bool {
     // Cannot be implemented for TTS
     return false
@@ -172,7 +171,7 @@ public class FlutterTTSNavigator: FlutterTimebasedNavigator, PublicationSpeechSy
   }
 
   func ttsSetVoice(voiceIdentifier: String) throws {
-    print(TAG, "ttsSetVoice: voiceIdent=\(String(describing: voiceIdentifier))")
+    debugPrint(TAG, "ttsSetVoice: voiceIdent=\(String(describing: voiceIdentifier))")
 
     /// Check that voice with given identifier exists
     guard let _ = synthesizer?.voiceWithIdentifier(voiceIdentifier) else {
@@ -186,23 +185,23 @@ public class FlutterTTSNavigator: FlutterTimebasedNavigator, PublicationSpeechSy
   // MARK: PublicationSpeechSynthesizerDelegate
 
   public func publicationSpeechSynthesizer(_ synthesizer: ReadiumNavigator.PublicationSpeechSynthesizer, stateDidChange state: ReadiumNavigator.PublicationSpeechSynthesizer.State) {
-    print(TAG, "publicationSpeechSynthesizerStateDidChange")
+    debugPrint(TAG, "publicationSpeechSynthesizerStateDidChange")
 
     switch state {
     case let .playing(utt, wordRange):
-      print(TAG, "tts playing")
+      debugPrint(TAG, "tts playing")
       /// utterance is a full sentence/paragraph, while range is the currently spoken part.
       playingUtterance = utt.locator
       if let wordRange = wordRange {
         playingWordRangeSubject.send(wordRange)
       }
       self.listener?.timebasedNavigator(self, requestsHighlightAt: utt.locator, withWordLocator: wordRange)
     case let .paused(utt):
-      print(TAG, "tts paused at utterance: \(utt.text)")
+      debugPrint(TAG, "tts paused at utterance: \(utt.text)")
       playingUtterance = utt.locator
     case .stopped:
       playingUtterance = nil
-      print(TAG, "tts stopped")
+      debugPrint(TAG, "tts stopped")
       self.listener?.timebasedNavigator(self, requestsHighlightAt: nil, withWordLocator: nil)
       //updateDecorations(uttLocator: nil, rangeLocator: nil)
       self.nowPlayingUpdater.clearNowPlaying()
@@ -213,7 +212,7 @@ public class FlutterTTSNavigator: FlutterTimebasedNavigator, PublicationSpeechSy
   }
 
   public func publicationSpeechSynthesizer(_ synthesizer: ReadiumNavigator.PublicationSpeechSynthesizer, utterance: ReadiumNavigator.PublicationSpeechSynthesizer.Utterance, didFailWithError error: ReadiumNavigator.PublicationSpeechSynthesizer.Error) {
-    print(TAG, "publicationSpeechSynthesizerUtteranceDidFail: \(error)")
+    debugPrint(TAG, "publicationSpeechSynthesizerUtteranceDidFail: \(error)")
 
     self.listener?.timebasedNavigator(self, encounteredError: error, withDescription: "TTSUtteranceFailed")