From d9dc9baa5106aa65a2d0d71d6669ff369199dbe4 Mon Sep 17 00:00:00 2001 From: jenstandstad Date: Wed, 5 Aug 2026 12:46:51 +0200 Subject: [PATCH] Fixing the audio looping and pause narration issue --- gnommo/models.py | 8 ++++ gnommo/renderer.py | 88 +++++++++++++++++++++++++++++---------- gnommo/transformer.py | 42 +++++++++++++++---- gnommo/validator.py | 7 ++-- tests/test_chunking_v2.py | 22 ++++++++++ 5 files changed, 134 insertions(+), 33 deletions(-) diff --git a/gnommo/models.py b/gnommo/models.py index 6e53c53..d838042 100644 --- a/gnommo/models.py +++ b/gnommo/models.py @@ -429,6 +429,14 @@ class AudioEvent: # position (seconds) into the source stream to begin at — the loop phase for # looping music, or a linear seek for one-shots. 0.0 = play from the start (v1). src_offset: float = 0.0 + # Loop phase for the CROSSFADE loop path specifically. That stream is periodic + # with period (duration - overlap), not `duration`, so its seam-resume phase is + # `elapsed % (duration - overlap)` — a different modulus than src_offset. Only + # set for looping clips that define an overlap; 0.0 otherwise. + crossfade_offset: float = 0.0 + # Explicit stop time (output timeline) from an [end:handle] marker; None means + # play to the render/window end (loop) or the clip's natural length (one-shot). + end_time: Optional[float] = None @dataclass diff --git a/gnommo/renderer.py b/gnommo/renderer.py index b4265c6..317fce5 100644 --- a/gnommo/renderer.py +++ b/gnommo/renderer.py @@ -78,6 +78,7 @@ def _build_crossfade_loop_filter( needed_duration: float, volume: float, delay_ms: int, + start_offset: float = 0.0, ) -> list[str]: """ Build FFmpeg filter chain for crossfade looping. @@ -85,6 +86,12 @@ def _build_crossfade_loop_filter( Creates a seamless loop by overlapping copies of the audio with fade in/out. Each loop iteration crossfades with the next for `overlap` seconds. + The crossfaded stream is periodic with period ``loop_len = audio_duration - + overlap``. ``start_offset`` seeks into that continuous stream, so a chunk + that begins mid-loop (e.g. the second half of a partial/chunked render) + resumes at the correct loop phase instead of restarting from the top. This + is what keeps background music seamless across chunk seams. + Args: input_label: Input stream label (e.g., "[0:a]") output_label: Output stream label (e.g., "[aud0]") @@ -93,6 +100,7 @@ def _build_crossfade_loop_filter( needed_duration: Total duration needed volume: Volume multiplier delay_ms: Initial delay in milliseconds + start_offset: Phase (seconds) into the crossfade loop stream to start at Returns: List of filter strings to append to the filter_complex @@ -100,8 +108,12 @@ def _build_crossfade_loop_filter( filters = [] loop_len = audio_duration - overlap + # Build the crossfade stream from phase 0, long enough to cover the phase we + # seek past plus the duration we actually need, then trim [start_offset ...]. + build_duration = start_offset + needed_duration + # Calculate number of loop iterations needed (add 1 extra for safety) - n_loops = math.ceil(needed_duration / loop_len) + 1 + n_loops = math.ceil(build_duration / loop_len) + 1 # Limit to reasonable number of loops to avoid filter complexity explosion n_loops = min(n_loops, 100) @@ -109,7 +121,7 @@ def _build_crossfade_loop_filter( if n_loops <= 1: # Single play, no looping needed filters.append( - f"{input_label}atrim=0:{needed_duration:.3f}," + f"{input_label}atrim={start_offset:.3f}:{start_offset + needed_duration:.3f}," f"asetpts=PTS-STARTPTS," f"adelay={delay_ms}|{delay_ms}," f"volume={volume:.2f}{output_label}" @@ -120,15 +132,16 @@ def _build_crossfade_loop_filter( split_labels = [f"[xfloop_{output_label[1:-1]}_{i}]" for i in range(n_loops)] filters.append(f"{input_label}asplit={n_loops}{''.join(split_labels)}") - # Process each copy with appropriate delay and fades + # Process each copy with appropriate delay and fades. Copies are laid out in + # loop time (no output delay yet); the output delay/phase-trim is applied + # once after mixing so the phase seek is straightforward. mix_labels = [] for i in range(n_loops): copy_label = split_labels[i] out_label = f"[xfl_{output_label[1:-1]}_{i}]" mix_labels.append(out_label) - loop_delay = i * loop_len - total_delay_ms = delay_ms + int(loop_delay * 1000) + loop_delay_ms = int(i * loop_len * 1000) # Build filter chain for this copy chain_parts = [] @@ -143,17 +156,20 @@ def _build_crossfade_loop_filter( if fade_out_start > 0: chain_parts.append(f"afade=t=out:st={fade_out_start:.3f}:d={overlap:.3f}") - chain_parts.append(f"adelay={total_delay_ms}|{total_delay_ms}") - chain_parts.append(f"volume={volume:.2f}") + if loop_delay_ms > 0: + chain_parts.append(f"adelay={loop_delay_ms}|{loop_delay_ms}") filter_chain = ",".join(chain_parts) filters.append(f"{copy_label}{filter_chain}{out_label}") - # Mix all copies together, then trim to needed duration + # Mix all copies into the continuous loop stream, seek to the loop phase + # (start_offset), apply volume, then the output delay. filters.append( f"{''.join(mix_labels)}amix=inputs={n_loops}:duration=longest:normalize=0," - f"atrim=0:{needed_duration + delay_ms/1000:.3f}," - f"asetpts=PTS-STARTPTS{output_label}" + f"atrim={start_offset:.3f}:{start_offset + needed_duration:.3f}," + f"asetpts=PTS-STARTPTS," + f"volume={volume:.2f}," + f"adelay={delay_ms}|{delay_ms}{output_label}" ) return filters @@ -1487,11 +1503,16 @@ def build_filter_complex( for i, event in enumerate(plan.audio_events): audio_idx = audio_inputs[event.audio_id] volume = event.audio_def.volume + # An [end:handle] marker caps this clip's stop time; otherwise a loop + # fills to the render/window end and a one-shot plays its natural length. + _clip_end = getattr(event, "end_time", None) if event.audio_def.loop: - # Looping audio: loop source, then trim/segment - # Stop at narration end if there's an outro - loop_end_time = audio_end_time + # Looping audio: loop source, then trim/segment. Stop at the end + # marker if set, else at narration end / outro. + loop_end_time = ( + audio_end_time if _clip_end is None else min(audio_end_time, _clip_end) + ) remaining = loop_end_time - event.start_time if plan.narration_pauses and not event.audio_def.ignore_pauses: @@ -1557,6 +1578,10 @@ def build_filter_complex( needed_duration=remaining, volume=volume, delay_ms=delay_ms, + # Chunking v2: resume at the loop phase so background + # music continues across chunk seams instead of + # restarting from the top. + start_offset=getattr(event, "crossfade_offset", 0.0), ) filters.extend(crossfade_filters) else: @@ -1593,7 +1618,14 @@ def build_filter_complex( ) if not relevant_pauses: delay_ms = int(event.start_time * 1000) - _seek = f"atrim={_off:.3f},asetpts=PTS-STARTPTS," if _off > 0 else "" + if _clip_end is not None: + # [end:handle] → play only up to the stop time, then trim. + _dur = max(0.0, _clip_end - event.start_time) + _seek = f"atrim={_off:.3f}:{_off + _dur:.3f},asetpts=PTS-STARTPTS," + elif _off > 0: + _seek = f"atrim={_off:.3f},asetpts=PTS-STARTPTS," + else: + _seek = "" filters.append( f"[{audio_idx}:a]{_seek}adelay={delay_ms}|{delay_ms},volume={volume:.2f}[{label}]" ) @@ -1601,11 +1633,15 @@ def build_filter_complex( else: # Play [seg_start, pause) of source, freeze during the pause, # then resume — source position (src_pos) never advances across - # the gap. Final segment runs to the source's natural end. + # the gap. Final segment runs to the [end:handle] stop (if set), + # otherwise the source's natural end. + _end = _clip_end if _clip_end is not None else float("inf") src_pos = _off seg_start = event.start_time seg_count = 0 for pause in relevant_pauses: + if pause.output_time >= _end: + break if pause.output_time > seg_start: seg_dur = pause.output_time - seg_start seg_label = f"{label}_seg{seg_count}" @@ -1619,14 +1655,20 @@ def build_filter_complex( src_pos += seg_dur seg_count += 1 seg_start = pause.output_time + pause.duration - seg_label = f"{label}_seg{seg_count}" - d_ms = int(seg_start * 1000) - filters.append( - f"[{audio_idx}:a]atrim={src_pos:.3f}," - f"asetpts=PTS-STARTPTS,adelay={d_ms}|{d_ms}," - f"volume={volume:.2f}[{seg_label}]" - ) - audio_labels_to_mix.append(f"[{seg_label}]") + if seg_start < _end: + seg_label = f"{label}_seg{seg_count}" + d_ms = int(seg_start * 1000) + _atrim = ( + f"atrim={src_pos:.3f}:{src_pos + (_end - seg_start):.3f}" + if _clip_end is not None + else f"atrim={src_pos:.3f}" + ) + filters.append( + f"[{audio_idx}:a]{_atrim}," + f"asetpts=PTS-STARTPTS,adelay={d_ms}|{d_ms}," + f"volume={volume:.2f}[{seg_label}]" + ) + audio_labels_to_mix.append(f"[{seg_label}]") # Extract and mix audio from triggered video events _have_audio = video_events_with_audio or set() diff --git a/gnommo/transformer.py b/gnommo/transformer.py index 71e58e0..b1dad9c 100644 --- a/gnommo/transformer.py +++ b/gnommo/transformer.py @@ -964,6 +964,8 @@ def build_render_plan( event.end_time -= time_offset for event in audio_events: event.start_time = max(0, event.start_time - time_offset) + if event.end_time is not None: + event.end_time = max(0.0, event.end_time - time_offset) for event in camera_events: event.time -= time_offset @@ -1015,6 +1017,8 @@ def build_render_plan( for aud_event in audio_events: if aud_event.start_time > narration_time: aud_event.start_time += pause_duration + if aud_event.end_time is not None and aud_event.end_time > narration_time: + aud_event.end_time += pause_duration for cam_event in camera_events: if cam_event.time > narration_time: @@ -1438,9 +1442,12 @@ def _extract_video_events( if vt > start_time: end_time = vt break - # A pause-narration video must stay for at least the pause it holds. + # A pause-narration cutscene fills EXACTLY the freeze it creates (its + # content length == pause_narration), so it ends when the freeze ends — + # not stretched to the next video, which would keep it overlaying the + # resumed narration afterwards. if pause_narration: - end_time = max(end_time, start_time + pause_narration) + end_time = start_time + pause_narration elif end_on in ("next_slide", "slide"): # End at next slide marker ("slide" is a recognised alias for "next_slide") end_time = total_duration @@ -1448,10 +1455,9 @@ def _extract_video_events( if slide_time > start_time: end_time = slide_time break - # pause_narration videos must stay visible for the full pause duration — - # the narration is held for that long, so the overlay should match. + # pause_narration cutscene: end exactly with the freeze (see above). if pause_narration: - end_time = max(end_time, start_time + pause_narration) + end_time = start_time + pause_narration elif end_on == "end_marker": # Explicit end: stop at the first [end:handle] placed after this clip # starts (so the same handle can be reused in different sections). @@ -1471,7 +1477,7 @@ def _extract_video_events( f"after it — ending at the next video instead." ) if pause_narration: - end_time = max(end_time, start_time + pause_narration) + end_time = start_time + pause_narration else: # end_on None ([narration:] with no explicit end) — runs to end. end_time = total_duration @@ -1525,6 +1531,15 @@ def _extract_audio_events( range_start, range_end = time_range if time_range else (0.0, float("inf")) events: list[AudioEvent] = [] + # [end:handle] markers stop an audio clip early (parallel to the video end_marker, + # but audio opts in automatically — there is no per-clip end_on to set). + audio_end_markers: dict[str, list[float]] = {} + for timing in marker_timings: + if timing.timestamp is not None and timing.timestamp >= 0 and timing.marker_id.startswith("end:"): + audio_end_markers.setdefault(timing.marker_id[4:].lower(), []).append(timing.timestamp) + for _h in audio_end_markers: + audio_end_markers[_h].sort() + for timing in marker_timings: if timing.timestamp < 0: continue @@ -1538,8 +1553,13 @@ def _extract_audio_events( if audio_id is not None and audio_id in audio: adef = audio[audio_id] astart = max(0, timing.timestamp - AUDIO_OFFSET_SECONDS) + # Explicit stop from the first [end:audio_id] placed after this clip starts. + _ends = [t for t in audio_end_markers.get(audio_id.lower(), ()) if t > timing.timestamp] + clip_end = _ends[0] if _ends else None # Effective end of this clip on the output timeline. - if adef.loop: + if clip_end is not None: + aend = clip_end + elif adef.loop: aend = range_end # a loop fills to the window/render end elif adef.duration is not None: aend = astart + adef.duration @@ -1552,10 +1572,16 @@ def _extract_audio_events( if aend <= range_start or astart >= range_end: continue src_offset = 0.0 + crossfade_offset = 0.0 if astart < range_start: into = range_start - astart if adef.loop and adef.duration: src_offset = into % adef.duration + # The crossfade loop stream repeats every (duration - overlap), + # so it resumes at a different phase than the hard aloop path. + if adef.overlap: + loop_len = max(1e-6, adef.duration - adef.overlap) + crossfade_offset = into % loop_len else: src_offset = into astart = range_start @@ -1565,6 +1591,8 @@ def _extract_audio_events( start_time=astart, audio_def=adef, src_offset=src_offset, + crossfade_offset=crossfade_offset, + end_time=clip_end, ) ) diff --git a/gnommo/validator.py b/gnommo/validator.py index c6bb63d..5967db0 100644 --- a/gnommo/validator.py +++ b/gnommo/validator.py @@ -145,13 +145,14 @@ def validate_project( if marker in ("pause", "stop"): continue - # Explicit end markers: [end:handle] stops a video started with end_on=end_marker. + # Explicit end markers: [end:handle] stops a video (end_on=end_marker) OR an + # audio clip. Valid if the handle is defined in either videos.json or audio.json. if marker.startswith("end:"): handle = marker[4:].lower() - if handle not in videos: + if handle not in videos and handle not in (audio or {}): warnings.append( ValidationIssue( - f"[{marker}] ends a video, but '{handle}' isn't defined in videos.json.", + f"[{marker}] ends a clip, but '{handle}' isn't defined in videos.json or audio.json.", project_path / "manuscript.txt", ) ) diff --git a/tests/test_chunking_v2.py b/tests/test_chunking_v2.py index 74fd744..645fcd7 100644 --- a/tests/test_chunking_v2.py +++ b/tests/test_chunking_v2.py @@ -71,6 +71,27 @@ def test_audio(): check("full render includes music with no seek", "music" in full and full["music"].src_offset == 0.0) +def test_crossfade_phase(): + print("crossfade loop phase:") + # Looping pad with a 15s crossfade overlap: the crossfade stream repeats every + # (duration - overlap) = 60 - 15 = 45s, so a chunk starting at into=300 resumes + # at crossfade phase 300 % 45 = 30, while the hard-loop phase is 300 % 60 = 60→0. + audio = {"pad": AudioDefinition(file="pad.wav", loop=True, duration=60.0, overlap=15.0)} + markers = [MarkerTiming(marker_id="Apad", timestamp=0.0, context="", confidence=1.0)] + evs = {e.audio_id: e for e in _extract_audio_events(markers, audio, time_range=(300.0, 600.0))} + check("looping pad with overlap is included", "pad" in evs) + if "pad" in evs: + p = evs["pad"] + check("crossfade seeks to loop_len phase 30.0", + abs(p.crossfade_offset - 30.0) < 1e-6, f"crossfade_offset={p.crossfade_offset}") + check("src_offset still uses full-duration phase 0.0", + abs(p.src_offset - 0.0) < 1e-6, f"src_offset={p.src_offset}") + # Full render: no phase seek on either. + full = {e.audio_id: e for e in _extract_audio_events(markers, audio, time_range=None)} + check("full render pad has no crossfade seek", + "pad" in full and full["pad"].crossfade_offset == 0.0) + + # ── video ──────────────────────────────────────────────────────────────────── def test_video(): print("video events:") @@ -115,6 +136,7 @@ def test_video(): if __name__ == "__main__": test_audio() + test_crossfade_phase() test_video() print() if _fails: