gitprov/gnommo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixing the audio looping and pause narration issue

Browse Source
This commit is contained in:
gitprov
2026-08-05 12:46:51 +02:00
parent f328759eab
commit d9dc9baa51
5 changed files with 134 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gnommo/models.py
+8
View File
@@ -429,6 +429,14 @@ class AudioEvent:
# position (seconds) into the source stream to begin at — the loop phase for
# looping music, or a linear seek for one-shots. 0.0 = play from the start (v1).
src_offset: float = 0.0
# Loop phase for the CROSSFADE loop path specifically. That stream is periodic
# with period (duration - overlap), not `duration`, so its seam-resume phase is
# `elapsed % (duration - overlap)` — a different modulus than src_offset. Only
# set for looping clips that define an overlap; 0.0 otherwise.
crossfade_offset: float = 0.0
# Explicit stop time (output timeline) from an [end:handle] marker; None means
# play to the render/window end (loop) or the clip's natural length (one-shot).
end_time: Optional[float] = None
@dataclass
gnommo/renderer.py
+65 -23
View File
@@ -78,6 +78,7 @@ def _build_crossfade_loop_filter(
needed_duration: float,
volume: float,
delay_ms: int,
start_offset: float = 0.0,
) -> list[str]:
"""
Build FFmpeg filter chain for crossfade looping.
@@ -85,6 +86,12 @@ def _build_crossfade_loop_filter(
Creates a seamless loop by overlapping copies of the audio with fade in/out.
Each loop iteration crossfades with the next for `overlap` seconds.
The crossfaded stream is periodic with period ``loop_len = audio_duration -
overlap``. ``start_offset`` seeks into that continuous stream, so a chunk
that begins mid-loop (e.g. the second half of a partial/chunked render)
resumes at the correct loop phase instead of restarting from the top. This
is what keeps background music seamless across chunk seams.
Args:
input_label: Input stream label (e.g., "[0:a]")
output_label: Output stream label (e.g., "[aud0]")
@@ -93,6 +100,7 @@ def _build_crossfade_loop_filter(
needed_duration: Total duration needed
volume: Volume multiplier
delay_ms: Initial delay in milliseconds
start_offset: Phase (seconds) into the crossfade loop stream to start at
Returns:
List of filter strings to append to the filter_complex
@@ -100,8 +108,12 @@ def _build_crossfade_loop_filter(
filters = []
loop_len = audio_duration - overlap
# Build the crossfade stream from phase 0, long enough to cover the phase we
# seek past plus the duration we actually need, then trim [start_offset ...].
build_duration = start_offset + needed_duration
# Calculate number of loop iterations needed (add 1 extra for safety)
n_loops = math.ceil(needed_duration / loop_len) + 1
n_loops = math.ceil(build_duration / loop_len) + 1
# Limit to reasonable number of loops to avoid filter complexity explosion
n_loops = min(n_loops, 100)
@@ -109,7 +121,7 @@ def _build_crossfade_loop_filter(
if n_loops <= 1:
# Single play, no looping needed
filters.append(
f"{input_label}atrim=0:{needed_duration:.3f},"
f"{input_label}atrim={start_offset:.3f}:{start_offset + needed_duration:.3f},"
f"asetpts=PTS-STARTPTS,"
f"adelay={delay_ms}|{delay_ms},"
f"volume={volume:.2f}{output_label}"
@@ -120,15 +132,16 @@ def _build_crossfade_loop_filter(
split_labels = [f"[xfloop_{output_label[1:-1]}_{i}]" for i in range(n_loops)]
filters.append(f"{input_label}asplit={n_loops}{''.join(split_labels)}")
# Process each copy with appropriate delay and fades
# Process each copy with appropriate delay and fades. Copies are laid out in
# loop time (no output delay yet); the output delay/phase-trim is applied
# once after mixing so the phase seek is straightforward.
mix_labels = []
for i in range(n_loops):
copy_label = split_labels[i]
out_label = f"[xfl_{output_label[1:-1]}_{i}]"
mix_labels.append(out_label)
loop_delay = i * loop_len
total_delay_ms = delay_ms + int(loop_delay * 1000)
loop_delay_ms = int(i * loop_len * 1000)
# Build filter chain for this copy
chain_parts = []
@@ -143,17 +156,20 @@ def _build_crossfade_loop_filter(
if fade_out_start > 0:
chain_parts.append(f"afade=t=out:st={fade_out_start:.3f}:d={overlap:.3f}")
chain_parts.append(f"adelay={total_delay_ms}|{total_delay_ms}")
chain_parts.append(f"volume={volume:.2f}")
if loop_delay_ms > 0:
chain_parts.append(f"adelay={loop_delay_ms}|{loop_delay_ms}")
filter_chain = ",".join(chain_parts)
filters.append(f"{copy_label}{filter_chain}{out_label}")
# Mix all copies together, then trim to needed duration
# Mix all copies into the continuous loop stream, seek to the loop phase
# (start_offset), apply volume, then the output delay.
filters.append(
f"{''.join(mix_labels)}amix=inputs={n_loops}:duration=longest:normalize=0,"
f"atrim=0:{needed_duration + delay_ms/1000:.3f},"
f"asetpts=PTS-STARTPTS{output_label}"
f"atrim={start_offset:.3f}:{start_offset + needed_duration:.3f},"
f"asetpts=PTS-STARTPTS,"
f"volume={volume:.2f},"
f"adelay={delay_ms}|{delay_ms}{output_label}"
)
return filters
@@ -1487,11 +1503,16 @@ def build_filter_complex(
for i, event in enumerate(plan.audio_events):
audio_idx = audio_inputs[event.audio_id]
volume = event.audio_def.volume
# An [end:handle] marker caps this clip's stop time; otherwise a loop
# fills to the render/window end and a one-shot plays its natural length.
_clip_end = getattr(event, "end_time", None)
if event.audio_def.loop:
# Looping audio: loop source, then trim/segment
# Stop at narration end if there's an outro
loop_end_time = audio_end_time
# Looping audio: loop source, then trim/segment. Stop at the end
# marker if set, else at narration end / outro.
loop_end_time = (
audio_end_time if _clip_end is None else min(audio_end_time, _clip_end)
)
remaining = loop_end_time - event.start_time
if plan.narration_pauses and not event.audio_def.ignore_pauses:
@@ -1557,6 +1578,10 @@ def build_filter_complex(
needed_duration=remaining,
volume=volume,
delay_ms=delay_ms,
# Chunking v2: resume at the loop phase so background
# music continues across chunk seams instead of
# restarting from the top.
start_offset=getattr(event, "crossfade_offset", 0.0),
)
filters.extend(crossfade_filters)
else:
@@ -1593,7 +1618,14 @@ def build_filter_complex(
)
if not relevant_pauses:
delay_ms = int(event.start_time * 1000)
_seek = f"atrim={_off:.3f},asetpts=PTS-STARTPTS," if _off > 0 else ""
if _clip_end is not None:
# [end:handle] → play only up to the stop time, then trim.
_dur = max(0.0, _clip_end - event.start_time)
_seek = f"atrim={_off:.3f}:{_off + _dur:.3f},asetpts=PTS-STARTPTS,"
elif _off > 0:
_seek = f"atrim={_off:.3f},asetpts=PTS-STARTPTS,"
else:
_seek = ""
filters.append(
f"[{audio_idx}:a]{_seek}adelay={delay_ms}|{delay_ms},volume={volume:.2f}[{label}]"
)
@@ -1601,11 +1633,15 @@ def build_filter_complex(
else:
# Play [seg_start, pause) of source, freeze during the pause,
# then resume — source position (src_pos) never advances across
# the gap. Final segment runs to the source's natural end.
# the gap. Final segment runs to the [end:handle] stop (if set),
# otherwise the source's natural end.
_end = _clip_end if _clip_end is not None else float("inf")
src_pos = _off
seg_start = event.start_time
seg_count = 0
for pause in relevant_pauses:
if pause.output_time >= _end:
break
if pause.output_time > seg_start:
seg_dur = pause.output_time - seg_start
seg_label = f"{label}_seg{seg_count}"
@@ -1619,14 +1655,20 @@ def build_filter_complex(
src_pos += seg_dur
seg_count += 1
seg_start = pause.output_time + pause.duration
seg_label = f"{label}_seg{seg_count}"
d_ms = int(seg_start * 1000)
filters.append(
f"[{audio_idx}:a]atrim={src_pos:.3f},"
f"asetpts=PTS-STARTPTS,adelay={d_ms}|{d_ms},"
f"volume={volume:.2f}[{seg_label}]"
)
audio_labels_to_mix.append(f"[{seg_label}]")
if seg_start < _end:
seg_label = f"{label}_seg{seg_count}"
d_ms = int(seg_start * 1000)
_atrim = (
f"atrim={src_pos:.3f}:{src_pos + (_end - seg_start):.3f}"
if _clip_end is not None
else f"atrim={src_pos:.3f}"
)
filters.append(
f"[{audio_idx}:a]{_atrim},"
f"asetpts=PTS-STARTPTS,adelay={d_ms}|{d_ms},"
f"volume={volume:.2f}[{seg_label}]"
)
audio_labels_to_mix.append(f"[{seg_label}]")
# Extract and mix audio from triggered video events
_have_audio = video_events_with_audio or set()
gnommo/transformer.py
+35 -7
View File
@@ -964,6 +964,8 @@ def build_render_plan(
event.end_time -= time_offset
for event in audio_events:
event.start_time = max(0, event.start_time - time_offset)
if event.end_time is not None:
event.end_time = max(0.0, event.end_time - time_offset)
for event in camera_events:
event.time -= time_offset
@@ -1015,6 +1017,8 @@ def build_render_plan(
for aud_event in audio_events:
if aud_event.start_time > narration_time:
aud_event.start_time += pause_duration
if aud_event.end_time is not None and aud_event.end_time > narration_time:
aud_event.end_time += pause_duration
for cam_event in camera_events:
if cam_event.time > narration_time:
@@ -1438,9 +1442,12 @@ def _extract_video_events(
if vt > start_time:
end_time = vt
break
# A pause-narration video must stay for at least the pause it holds.
# A pause-narration cutscene fills EXACTLY the freeze it creates (its
# content length == pause_narration), so it ends when the freeze ends —
# not stretched to the next video, which would keep it overlaying the
# resumed narration afterwards.
if pause_narration:
end_time = max(end_time, start_time + pause_narration)
end_time = start_time + pause_narration
elif end_on in ("next_slide", "slide"):
# End at next slide marker ("slide" is a recognised alias for "next_slide")
end_time = total_duration
@@ -1448,10 +1455,9 @@ def _extract_video_events(
if slide_time > start_time:
end_time = slide_time
break
# pause_narration videos must stay visible for the full pause duration —
# the narration is held for that long, so the overlay should match.
# pause_narration cutscene: end exactly with the freeze (see above).
if pause_narration:
end_time = max(end_time, start_time + pause_narration)
end_time = start_time + pause_narration
elif end_on == "end_marker":
# Explicit end: stop at the first [end:handle] placed after this clip
# starts (so the same handle can be reused in different sections).
@@ -1471,7 +1477,7 @@ def _extract_video_events(
f"after it — ending at the next video instead."
)
if pause_narration:
end_time = max(end_time, start_time + pause_narration)
end_time = start_time + pause_narration
else:
# end_on None ([narration:] with no explicit end) — runs to end.
end_time = total_duration
@@ -1525,6 +1531,15 @@ def _extract_audio_events(
range_start, range_end = time_range if time_range else (0.0, float("inf"))
events: list[AudioEvent] = []
# [end:handle] markers stop an audio clip early (parallel to the video end_marker,
# but audio opts in automatically — there is no per-clip end_on to set).
audio_end_markers: dict[str, list[float]] = {}
for timing in marker_timings:
if timing.timestamp is not None and timing.timestamp >= 0 and timing.marker_id.startswith("end:"):
audio_end_markers.setdefault(timing.marker_id[4:].lower(), []).append(timing.timestamp)
for _h in audio_end_markers:
audio_end_markers[_h].sort()
for timing in marker_timings:
if timing.timestamp < 0:
continue
@@ -1538,8 +1553,13 @@ def _extract_audio_events(
if audio_id is not None and audio_id in audio:
adef = audio[audio_id]
astart = max(0, timing.timestamp - AUDIO_OFFSET_SECONDS)
# Explicit stop from the first [end:audio_id] placed after this clip starts.
_ends = [t for t in audio_end_markers.get(audio_id.lower(), ()) if t > timing.timestamp]
clip_end = _ends[0] if _ends else None
# Effective end of this clip on the output timeline.
if adef.loop:
if clip_end is not None:
aend = clip_end
elif adef.loop:
aend = range_end # a loop fills to the window/render end
elif adef.duration is not None:
aend = astart + adef.duration
@@ -1552,10 +1572,16 @@ def _extract_audio_events(
if aend <= range_start or astart >= range_end:
continue
src_offset = 0.0
crossfade_offset = 0.0
if astart < range_start:
into = range_start - astart
if adef.loop and adef.duration:
src_offset = into % adef.duration
# The crossfade loop stream repeats every (duration - overlap),
# so it resumes at a different phase than the hard aloop path.
if adef.overlap:
loop_len = max(1e-6, adef.duration - adef.overlap)
crossfade_offset = into % loop_len
else:
src_offset = into
astart = range_start
@@ -1565,6 +1591,8 @@ def _extract_audio_events(
start_time=astart,
audio_def=adef,
src_offset=src_offset,
crossfade_offset=crossfade_offset,
end_time=clip_end,
)
)
gnommo/validator.py
+4 -3
View File
@@ -145,13 +145,14 @@ def validate_project(
if marker in ("pause", "stop"):
continue
# Explicit end markers: [end:handle] stops a video started with end_on=end_marker.
# Explicit end markers: [end:handle] stops a video (end_on=end_marker) OR an
# audio clip. Valid if the handle is defined in either videos.json or audio.json.
if marker.startswith("end:"):
handle = marker[4:].lower()
if handle not in videos:
if handle not in videos and handle not in (audio or {}):
warnings.append(
ValidationIssue(
f"[{marker}] ends a video, but '{handle}' isn't defined in videos.json.",
f"[{marker}] ends a clip, but '{handle}' isn't defined in videos.json or audio.json.",
project_path / "manuscript.txt",
)
)
tests/test_chunking_v2.py
+22
View File
@@ -71,6 +71,27 @@ def test_audio():
check("full render includes music with no seek", "music" in full and full["music"].src_offset == 0.0)
def test_crossfade_phase():
print("crossfade loop phase:")
# Looping pad with a 15s crossfade overlap: the crossfade stream repeats every
# (duration - overlap) = 60 - 15 = 45s, so a chunk starting at into=300 resumes
# at crossfade phase 300 % 45 = 30, while the hard-loop phase is 300 % 60 = 60→0.
audio = {"pad": AudioDefinition(file="pad.wav", loop=True, duration=60.0, overlap=15.0)}
markers = [MarkerTiming(marker_id="Apad", timestamp=0.0, context="", confidence=1.0)]
evs = {e.audio_id: e for e in _extract_audio_events(markers, audio, time_range=(300.0, 600.0))}
check("looping pad with overlap is included", "pad" in evs)
if "pad" in evs:
p = evs["pad"]
check("crossfade seeks to loop_len phase 30.0",
abs(p.crossfade_offset - 30.0) < 1e-6, f"crossfade_offset={p.crossfade_offset}")
check("src_offset still uses full-duration phase 0.0",
abs(p.src_offset - 0.0) < 1e-6, f"src_offset={p.src_offset}")
# Full render: no phase seek on either.
full = {e.audio_id: e for e in _extract_audio_events(markers, audio, time_range=None)}
check("full render pad has no crossfade seek",
"pad" in full and full["pad"].crossfade_offset == 0.0)
# ── video ────────────────────────────────────────────────────────────────────
def test_video():
print("video events:")
@@ -115,6 +136,7 @@ def test_video():
if __name__ == "__main__":
test_audio()
test_crossfade_phase()
test_video()
print()
if _fails: