Skip to content

Commit c2ae3b8

Browse files
committed
feat(editor): imported audio as a first-class timeline region
The audio-import feature (issue #350): clip-anchored audio regions on dedicated lanes, waveform pills with drag/resize/gain, contiguous preview playback matching the export, compositor mixing on all three platforms, agent addAudio/setAudio tools, and the two review passes folded in (005bec7/efd1e058 then 49d543f in the pre-rewrite history). Linearized onto claude/pill-keyboard-activation (updated with main) so the PR can take a rebase merge: the previous history carried two merge commits of main inside the branch, which GitHub cannot replay onto the base.
1 parent 375da38 commit c2ae3b8

117 files changed

Lines changed: 4551 additions & 198 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎crates/compositor/src/audio.rs‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
use crate::ffi::*;
66

77
use crate::regions::SpeedSegment;
8-
use crate::scene::SceneAudio;
8+
use crate::scene::{SceneAudio, SceneAudioTrack};
99
use anyhow::{bail, Result};
1010
use std::f32::consts::PI;
1111
use std::ffi::CString;
@@ -1381,6 +1381,84 @@ pub fn assemble_concatenated_pcm(
13811381
output
13821382
}
13831383

1384+
/// Mix imported audio tracks (issue #350) over the assembled programme.
1385+
///
1386+
/// Each track is decoded across its trim window — already resampled to 48 kHz
1387+
/// stereo by `decode_clip_audio`, the same path a clip's own audio takes — scaled
1388+
/// by its per-track gain (the same `10^(dB/20)` law as `finish_audio`), and summed
1389+
/// into the programme at `start_sec`. The programme length is NOT extended: a
1390+
/// track that runs past the video is truncated to it, so the audio and video
1391+
/// streams stay the same length for the muxer.
1392+
///
1393+
/// The decode window is capped up front at the room left in the programme after
1394+
/// `start_sec`, and a track starting at/after the end is skipped without decoding.
1395+
/// `decode_clip_audio` preallocates from the window, so this keeps a long track
1396+
/// pinned near a short programme's end from buffering (and clamping away) hours of
1397+
/// PCM. `trim_end_sec` must therefore be concrete — the renderer sends
1398+
/// `trimEnd ?? durationSec`.
1399+
///
1400+
/// A track whose file has no decodable audio is skipped — the same degradation a
1401+
/// stream-less clip gets.
1402+
pub fn mix_external_tracks(mut programme: PlanarPcm, tracks: &[SceneAudioTrack]) -> PlanarPcm {
1403+
let programme_len = programme.first().map(Vec::len).unwrap_or(0);
1404+
if programme_len == 0 {
1405+
return programme;
1406+
}
1407+
for track in tracks {
1408+
let offset = (track.start_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize;
1409+
// A track that starts at or past the programme end contributes nothing —
1410+
// skip it before decoding anything.
1411+
if offset >= programme_len {
1412+
continue;
1413+
}
1414+
let trim_start = track.trim_start_sec.max(0.0);
1415+
let Some(trim_end_full) = track.trim_end_sec else {
1416+
// Without a concrete end there is no safe window to decode (see the doc
1417+
// comment); the renderer always resolves one, so this only guards a
1418+
// hand-written scene.
1419+
continue;
1420+
};
1421+
// Cap the decode window at the room left in the programme. Everything past
1422+
// `offset` that overflows is discarded by `overlay_track_pcm` anyway, so
1423+
// decoding it only wastes time and memory — a three-hour track placed at
1424+
// second 9 of a ten-second export must not buffer three hours of PCM.
1425+
let remaining_sec = (programme_len - offset) as f64 / AUDIO_OUTPUT_SAMPLE_RATE as f64;
1426+
let trim_end = trim_end_full.min(trim_start + remaining_sec);
1427+
if trim_end <= trim_start {
1428+
continue;
1429+
}
1430+
let decoded = match decode_clip_audio(&track.path, trim_start, trim_end) {
1431+
Ok(Some(pcm)) => pcm,
1432+
_ => continue,
1433+
};
1434+
let gain = 10.0f32.powf(track.gain_db.clamp(-12.0, 12.0) / 20.0);
1435+
overlay_track_pcm(&mut programme, &decoded, offset, gain);
1436+
}
1437+
programme
1438+
}
1439+
1440+
/// Sum one decoded track into the programme at `offset` samples, scaled by `gain`,
1441+
/// truncated at the programme's end. Split out of `mix_external_tracks` so the
1442+
/// placement/gain/clamp math is testable without ffmpeg, exactly like
1443+
/// `mix_aligned_tracks` is split from the decode above.
1444+
fn overlay_track_pcm(programme: &mut PlanarPcm, decoded: &PlanarPcm, offset: usize, gain: f32) {
1445+
let programme_len = programme.first().map(Vec::len).unwrap_or(0);
1446+
if offset >= programme_len {
1447+
return;
1448+
}
1449+
let room = programme_len - offset;
1450+
for channel in 0..AUDIO_OUTPUT_CHANNELS {
1451+
let Some(source) = decoded.get(channel) else {
1452+
continue;
1453+
};
1454+
let count = source.len().min(room);
1455+
let dst = &mut programme[channel];
1456+
for k in 0..count {
1457+
dst[offset + k] += source[k] * gain;
1458+
}
1459+
}
1460+
}
1461+
13841462
/// Encodeur AAC attaché au muxer avant son header. Les paquets utilisent le même interleaver
13851463
/// que la vidéo ; les pts restent en unités échantillon jusqu'au rescale vers l'AVStream.
13861464
pub(crate) struct AacEncoder {
@@ -1692,6 +1770,64 @@ mod tests {
16921770
assert_eq!(mixed[1], vec![0.25, -0.5, 0.75]);
16931771
}
16941772

1773+
// Imported audio track overlay (issue #350).
1774+
#[test]
1775+
fn overlay_sums_at_offset_with_gain() {
1776+
let mut programme = planar(&[0.1, 0.1, 0.1, 0.1]);
1777+
// ×2 gain, placed at sample offset 1.
1778+
overlay_track_pcm(&mut programme, &planar(&[0.2, 0.2]), 1, 2.0);
1779+
assert_eq!(programme[0], vec![0.1, 0.5, 0.5, 0.1]);
1780+
assert_eq!(programme[1], vec![0.1, 0.5, 0.5, 0.1]);
1781+
}
1782+
1783+
#[test]
1784+
fn overlay_truncates_a_track_that_runs_past_the_programme() {
1785+
let mut programme = planar(&[0.0, 0.0, 0.0]);
1786+
// A 4-sample track placed at offset 2 has room for only 1 sample.
1787+
overlay_track_pcm(&mut programme, &planar(&[1.0, 1.0, 1.0, 1.0]), 2, 1.0);
1788+
assert_eq!(programme[0], vec![0.0, 0.0, 1.0]);
1789+
}
1790+
1791+
#[test]
1792+
fn overlay_past_the_end_is_a_no_op() {
1793+
let mut programme = planar(&[0.3, 0.3]);
1794+
overlay_track_pcm(&mut programme, &planar(&[1.0]), 5, 1.0);
1795+
assert_eq!(programme[0], vec![0.3, 0.3]);
1796+
}
1797+
1798+
#[test]
1799+
fn mix_external_tracks_skips_empty_windows() {
1800+
let programme = planar(&[0.4, 0.4]);
1801+
let tracks = vec![SceneAudioTrack {
1802+
path: "/nope.mp3".into(),
1803+
start_sec: 0.0,
1804+
gain_db: 0.0,
1805+
trim_start_sec: 2.0,
1806+
trim_end_sec: Some(1.0), // end <= start: empty window, never decoded
1807+
}];
1808+
// The empty window is skipped before any decode, so the programme is
1809+
// untouched even though the path does not exist.
1810+
let out = mix_external_tracks(programme, &tracks);
1811+
assert_eq!(out[0], vec![0.4, 0.4]);
1812+
}
1813+
1814+
#[test]
1815+
fn mix_external_tracks_skips_a_track_that_starts_past_the_programme() {
1816+
// 2 samples = ~0.00004 s of programme at 48 kHz; the track starts at 1 s, so
1817+
// its offset is past the end. It must be skipped before any decode is
1818+
// attempted (the path does not exist), never buffering its window.
1819+
let programme = planar(&[0.4, 0.4]);
1820+
let tracks = vec![SceneAudioTrack {
1821+
path: "/nope.mp3".into(),
1822+
start_sec: 1.0,
1823+
gain_db: 0.0,
1824+
trim_start_sec: 0.0,
1825+
trim_end_sec: Some(3600.0),
1826+
}];
1827+
let out = mix_external_tracks(programme, &tracks);
1828+
assert_eq!(out[0], vec![0.4, 0.4]);
1829+
}
1830+
16951831
#[test]
16961832
fn single_track_is_not_clamped() {
16971833
// Promesse de non-régression : une source mono-piste ressort telle quelle, y compris

‎crates/compositor/src/pipeline_linux.rs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use std::ffi::CString;
2121
use std::ptr;
2222

2323
use crate::audio::{
24-
assemble_concatenated_pcm, build_audio_concat_plan, finish_audio,
24+
assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
2525
AacEncoder, PlanarPcm,
2626
};
2727
use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -459,6 +459,12 @@ pub fn run_composited_multi(
459459

460460
let scene = comp.scene_snapshot();
461461
let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default();
462+
// Imported audio tracks (issue #350), cloned out of the borrowed scene so the
463+
// mix step below owns them. Empty for a project with no imported audio.
464+
let audio_tracks = scene
465+
.as_ref()
466+
.map(|scene| scene.audio_tracks.clone())
467+
.unwrap_or_default();
462468
// Ring de staging a 2 : l'export ne veut que du debit, une frame de latence
463469
// ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour
464470
// la raison pour laquelle la preview, elle, reste a 1.
@@ -552,7 +558,10 @@ pub fn run_composited_multi(
552558
let declared_audio: Vec<bool> = clips.iter().map(|c| c.has_audio).collect();
553559
let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64);
554560
audio_encoder.encode(
555-
&finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings),
561+
&finish_audio(
562+
mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
563+
audio_settings,
564+
),
556565
octx,
557566
)?;
558567
crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?;

‎crates/compositor/src/pipeline_macos.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
//! décodeurs, symétrique.
3131
3232
use crate::audio::{
33-
assemble_concatenated_pcm, build_audio_concat_plan, finish_audio,
33+
assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
3434
AacEncoder, PlanarPcm,
3535
};
3636
use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1078,6 +1078,11 @@ pub fn run_composited_multi(
10781078
// raconte avoir déjà coûté une fois.
10791079
let scene = comp.scene_snapshot();
10801080
let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default();
1081+
// Imported audio tracks (issue #350), cloned out of the borrowed scene.
1082+
let audio_tracks = scene
1083+
.as_ref()
1084+
.map(|scene| scene.audio_tracks.clone())
1085+
.unwrap_or_default();
10811086
frames = unsafe {
10821087
crate::timeline_walk::walk_composited_timeline(
10831088
clips,
@@ -1146,7 +1151,10 @@ pub fn run_composited_multi(
11461151
let declared_audio: Vec<bool> = clips.iter().map(|clip| clip.has_audio).collect();
11471152
let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64);
11481153
audio_encoder.encode(
1149-
&finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings),
1154+
&finish_audio(
1155+
mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
1156+
audio_settings,
1157+
),
11501158
octx,
11511159
)?;
11521160

‎crates/compositor/src/pipeline_windows.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
//! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps.
44
55
use crate::audio::{
6-
assemble_concatenated_pcm, build_audio_concat_plan, finish_audio,
6+
assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
77
AacEncoder, PlanarPcm,
88
};
99
use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1344,6 +1344,11 @@ unsafe fn run_multi_inner(
13441344
// fenêtrage par clip ; `walk_composited_timeline` s'en charge.
13451345
let scene = comp.scene_snapshot();
13461346
let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default();
1347+
// Imported audio tracks (issue #350), cloned out of the borrowed scene.
1348+
let audio_tracks = scene
1349+
.as_ref()
1350+
.map(|scene| scene.audio_tracks.clone())
1351+
.unwrap_or_default();
13471352

13481353
// ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ----
13491354
// Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur
@@ -1477,7 +1482,7 @@ unsafe fn run_multi_inner(
14771482
out_fps as f64,
14781483
);
14791484
let assembled_audio = finish_audio(
1480-
assemble_concatenated_pcm(&clip_pcm, &audio_plan),
1485+
mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks),
14811486
audio_settings,
14821487
);
14831488
audio_encoder.encode(&assembled_audio, octx)?;

‎crates/compositor/src/scene.rs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,29 @@ pub struct SceneAudio {
432432
pub gain_db: f32,
433433
}
434434

435+
/// One imported audio track (issue #350) mixed over the assembled programme —
436+
/// voiceover / BGM / SFX. Deliberately a SEPARATE `Scene` field rather than a
437+
/// member of `SceneAudio`, so `SceneAudio` stays `Copy` and the pipelines keep
438+
/// copying it out of a borrow unchanged.
439+
///
440+
/// `start_sec` is the track's head on the OUTPUT programme; `trim_start_sec` /
441+
/// `trim_end_sec` window the source file (both source seconds). The renderer
442+
/// resolves `start_sec` from the track's raw timeline position — equal to it when
443+
/// the project has no trims/speed, which is the case this first cut mixes exactly.
444+
#[derive(Debug, Clone, Default, Deserialize)]
445+
#[serde(rename_all = "camelCase")]
446+
pub struct SceneAudioTrack {
447+
pub path: String,
448+
#[serde(default)]
449+
pub start_sec: f64,
450+
#[serde(default)]
451+
pub gain_db: f32,
452+
#[serde(default)]
453+
pub trim_start_sec: f64,
454+
#[serde(default)]
455+
pub trim_end_sec: Option<f64>,
456+
}
457+
435458
#[derive(Debug, Clone, Copy, Deserialize)]
436459
#[serde(rename_all = "camelCase")]
437460
pub struct SceneOutput {
@@ -500,6 +523,10 @@ pub struct Scene {
500523
/// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible.
501524
#[serde(default)]
502525
pub audio: SceneAudio,
526+
/// Imported audio tracks mixed over the programme (issue #350). `#[serde(default)]`:
527+
/// absent from every scene written before this, and from a project with none.
528+
#[serde(default)]
529+
pub audio_tracks: Vec<SceneAudioTrack>,
503530
/// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS).
504531
#[serde(default)]
505532
pub crop_by_clip: Vec<Option<SceneCrop>>,

0 commit comments

Comments
 (0)