FrameAudioQueryの変更を反映できるようにする (#1961)

* レンダリング処理をリファクタリング、データ構造を変更

修正

* コメントを少し修正

* 歌い方と音声をキャッシュするように

* リファクタリング

* キャッシュ周りが正しく実装できていなかったので修正

* SingingStyleからSingingGuideに変更、コメントを追加

* コメントの追加・修正

* ブランド型にした

* singerAndFrameRateに変更

* hashとcalculatedHashに変更

* コメントを追加、処理を共通化

* レンダリング中に前の音声のキャッシュを再生しないように変更

* リファクタリング

* ノートシーケンスを作成して接続するタイミングを変更、コメントを修正

---------

Co-authored-by: Hiroshiba <hihokaruta@gmail.com>

src/components/Sing/ScoreSequencer.vue

@@ -229,9 +229,11 @@ import { isMac } from "@/type/preload";
 import { useStore } from "@/store";
 import { Note } from "@/store/type";
 import {
+  getEndTicksOfPhrase,
   getMeasureDuration,
   getNoteDuration,
   getNumOfMeasures,
+  getStartTicksOfPhrase,
 } from "@/sing/domain";
 import {
   getKeyBaseHeight,
@@ -372,8 +374,10 @@ const playheadX = computed(() => {
 // フレーズ
 const phraseInfos = computed(() => {
   return [...state.phrases.entries()].map(([key, phrase]) => {
-    const startBaseX = tickToBaseX(phrase.startTicks, tpqn.value);
-    const endBaseX = tickToBaseX(phrase.endTicks, tpqn.value);
+    const startTicks = getStartTicksOfPhrase(phrase);
+    const endTicks = getEndTicksOfPhrase(phrase);
+    const startBaseX = tickToBaseX(startTicks, tpqn.value);
+    const endBaseX = tickToBaseX(endTicks, tpqn.value);
     const startX = startBaseX * zoomX.value;
     const endX = endBaseX * zoomX.value;
     return { key, x: startX, width: endX - startX };

src/components/Sing/SequencerPitch.vue

@@ -3,7 +3,7 @@
 </template>
 
 <script setup lang="ts">
-import { ref, watch, toRaw, computed } from "vue";
+import { ref, watch, computed } from "vue";
 import * as PIXI from "pixi.js";
 import { useStore } from "@/store";
 import { frequencyToNoteNumber, secondToTick } from "@/sing/domain";
@@ -14,6 +14,7 @@ import {
   onMountedOrActivated,
   onUnmountedOrDeactivated,
 } from "@/composables/onMountOrActivate";
+import { SingingGuideSourceHash } from "@/store/type";
 
 type VoicedSection = {
   readonly startFrame: number;
@@ -34,8 +35,8 @@ const props = defineProps<{ offsetX: number; offsetY: number }>();
 
 const store = useStore();
 const queries = computed(() => {
-  const phrases = [...store.state.phrases.values()];
-  return phrases.map((value) => value.query);
+  const singingGuides = [...store.state.singingGuides.values()];
+  return singingGuides.map((value) => value.query);
 });
 
 const canvasContainer = ref<HTMLElement | null>(null);
@@ -48,7 +49,7 @@ let stage: PIXI.Container | undefined;
 let requestId: number | undefined;
 let renderInNextFrame = false;
 
-const pitchLinesMap = new Map<string, PitchLine[]>();
+const pitchLinesMap = new Map<SingingGuideSourceHash, PitchLine[]>();
 
 const searchVoicedSections = (phonemes: FramePhoneme[]) => {
   const voicedSections: VoicedSection[] = [];
@@ -89,35 +90,43 @@ const render = () => {
     throw new Error("stage is undefined.");
   }
 
-  const phrases = toRaw(store.state.phrases);
+  const tpqn = store.state.tpqn;
+  const tempos = [store.state.tempos[0]];
+  const singer = store.getters.SELECTED_TRACK.singer;
+  const singingGuides = store.state.singingGuides;
   const zoomX = store.state.sequencerZoomX;
   const zoomY = store.state.sequencerZoomY;
   const offsetX = props.offsetX;
   const offsetY = props.offsetY;
 
   // 無くなったフレーズを調べて、そのフレーズに対応するピッチラインを削除する
-  for (const [phraseKey, pitchLines] of pitchLinesMap) {
-    if (!phrases.has(phraseKey)) {
+  for (const [singingGuideKey, pitchLines] of pitchLinesMap) {
+    if (!singingGuides.has(singingGuideKey)) {
       for (const pitchLine of pitchLines) {
         stage.removeChild(pitchLine.lineStrip.displayObject);
         pitchLine.lineStrip.destroy();
       }
-      pitchLinesMap.delete(phraseKey);
+      pitchLinesMap.delete(singingGuideKey);
     }
   }
-  // ピッチラインの生成・更新を行う
-  for (const [phraseKey, phrase] of phrases) {
-    if (!phrase.singer || !phrase.query || phrase.startTime == undefined) {
-      continue;
+  // シンガーが未設定の場合はピッチラインをすべて非表示にして終了
+  if (!singer) {
+    for (const pitchLines of pitchLinesMap.values()) {
+      for (const pitchLine of pitchLines) {
+        pitchLine.lineStrip.renderable = false;
+      }
     }
-    const tempos = [toRaw(phrase.tempos[0])];
-    const tpqn = phrase.tpqn;
-    const startTime = phrase.startTime;
-    const f0 = phrase.query.f0;
-    const phonemes = phrase.query.phonemes;
-    const engineId = phrase.singer.engineId;
-    const frameRate = store.state.engineManifests[engineId].frameRate;
-    let pitchLines = pitchLinesMap.get(phraseKey);
+    renderer.render(stage);
+    return;
+  }
+  // ピッチラインの生成・更新を行う
+  for (const [singingGuideKey, singingGuide] of singingGuides) {
+    const f0 = singingGuide.query.f0;
+    const frameRate = singingGuide.frameRate;
+    const startTime = singingGuide.startTime;
+    const phonemes = singingGuide.query.phonemes;
+
+    let pitchLines = pitchLinesMap.get(singingGuideKey);
 
     // フレーズに対応するピッチラインが無かったら生成する
     if (!pitchLines) {
@@ -154,7 +163,7 @@ const render = () => {
       for (const pitchLine of pitchLines) {
         stage.addChild(pitchLine.lineStrip.displayObject);
       }
-      pitchLinesMap.set(phraseKey, pitchLines);
+      pitchLinesMap.set(singingGuideKey, pitchLines);
     }
 
     // ピッチラインを更新

src/sing/domain.ts

@@ -1,4 +1,15 @@
-import { Note, Phrase, Score, Tempo, TimeSignature } from "@/store/type";
+import { calculateHash } from "./utility";
+import {
+  Note,
+  Phrase,
+  Score,
+  SingingGuideSource,
+  SingingVoiceSource,
+  Tempo,
+  TimeSignature,
+  singingGuideSourceHashSchema,
+  singingVoiceSourceHashSchema,
+} from "@/store/type";
 
 const BEAT_TYPES = [2, 4, 8, 16];
 const MIN_BPM = 40;
@@ -295,9 +306,44 @@ export function isValidvolumeRangeAdjustment(volumeRangeAdjustment: number) {
   );
 }
 
+export const calculateNotesHash = async (notes: Note[]) => {
+  return await calculateHash({ notes });
+};
+
+export const calculateSingingGuideSourceHash = async (
+  singingGuideSource: SingingGuideSource
+) => {
+  const hash = await calculateHash(singingGuideSource);
+  return singingGuideSourceHashSchema.parse(hash);
+};
+
+export const calculateSingingVoiceSourceHash = async (
+  singingVoiceSource: SingingVoiceSource
+) => {
+  const hash = await calculateHash(singingVoiceSource);
+  return singingVoiceSourceHashSchema.parse(hash);
+};
+
+export function getStartTicksOfPhrase(phrase: Phrase) {
+  if (phrase.notes.length === 0) {
+    throw new Error("phrase.notes.length is 0.");
+  }
+  return phrase.notes[0].position;
+}
+
+export function getEndTicksOfPhrase(phrase: Phrase) {
+  if (phrase.notes.length === 0) {
+    throw new Error("phrase.notes.length is 0.");
+  }
+  const lastNote = phrase.notes[phrase.notes.length - 1];
+  return lastNote.position + lastNote.duration;
+}
+
 export function toSortedPhrases(phrases: Map<string, Phrase>) {
   return [...phrases.entries()].sort((a, b) => {
-    return a[1].startTicks - b[1].startTicks;
+    const startTicksOfPhraseA = getStartTicksOfPhrase(a[1]);
+    const startTicksOfPhraseB = getStartTicksOfPhrase(b[1]);
+    return startTicksOfPhraseA - startTicksOfPhraseB;
   });
 }
 
@@ -318,15 +364,18 @@ export function selectPriorPhrase(
   }
   // 再生位置が含まれるPhrase
   for (const [phraseKey, phrase] of phrases) {
-    if (phrase.startTicks <= position && position <= phrase.endTicks) {
+    if (
+      getStartTicksOfPhrase(phrase) <= position &&
+      position <= getEndTicksOfPhrase(phrase)
+    ) {
       return [phraseKey, phrase];
     }
   }
 
   const sortedPhrases = toSortedPhrases(phrases);
   // 再生位置より後のPhrase
   for (const [phraseKey, phrase] of sortedPhrases) {
-    if (phrase.startTicks > position) {
+    if (getStartTicksOfPhrase(phrase) > position) {
       return [phraseKey, phrase];
     }
   }

src/sing/storeHelper.ts

@@ -1,22 +1,10 @@
-import { Note, Singer, Tempo } from "@/store/type";
-import { generateHash } from "@/sing/utility";
+import { Note } from "@/store/type";
 
 export const DEFAULT_TPQN = 480;
 export const DEFAULT_BPM = 120;
 export const DEFAULT_BEATS = 4;
 export const DEFAULT_BEAT_TYPE = 4;
 
-export const generatePhraseHash = async (obj: {
-  singer: Singer | undefined;
-  keyRangeAdjustment: number;
-  volumeRangeAdjustment: number;
-  tpqn: number;
-  tempos: Tempo[];
-  notes: Note[];
-}) => {
-  return generateHash(obj);
-};
-
 /**
  * 頻繁に変更される値を保持します。
  * 値変更時に実行する関数を登録できます。

src/sing/utility.ts

@@ -3,7 +3,7 @@ export function round(value: number, digits: number) {
   return Math.round(value * powerOf10) / powerOf10;
 }
 
-export const generateHash = async <T>(obj: T) => {
+export const calculateHash = async <T>(obj: T) => {
   const textEncoder = new TextEncoder();
   const data = textEncoder.encode(JSON.stringify(obj));
   const digest = await crypto.subtle.digest("SHA-256", data);