From a8aab55bd27e5eaabb46c735c56168b2f231fdc9 Mon Sep 17 00:00:00 2001 From: jenstandstad Date: Sun, 26 Jul 2026 12:35:09 +0200 Subject: [PATCH] Chunking v2: keep clips that span a chunk boundary (overlap + seek) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1 dropped any overlay video or audio clip whose start fell outside a chunk's window, so a multi-slide background video — or looping background music — silently vanished from every chunk after the one it started in, diverging from a full render. v2 includes any clip that OVERLAPS the window and seeks into clips that began earlier so they resume mid-clip at the seam instead of restarting: - _extract_video_events: overlap test + VideoEvent.skip_override (loop-aware phase); the input -ss also seeks the clip's embedded audio (tvaud). - _extract_audio_events: overlap test + AudioEvent.src_offset (loop phase / linear); renderer honors it in the loop-with-pauses, standard-loop, and one-shot paths. - _chunk_boundary_span_warnings downgraded from "will be dropped" to informational. Full (non-chunked) render is unchanged: time_range=None leaves skip_override/ src_offset at defaults. Slides already used overlap; outros are last-chunk-only. Plan-level checks in tests/test_chunking_v2.py (all green). Remaining gate: render- seam frame/phase validation on the rig; crossfade-loop audio seek deferred. See docs/chunking_v2.md. Co-Authored-By: Claude Opus 4.8 --- docs/chunking_v2.md | 37 ++++++++---- gnommo/cli.py | 56 ++++++++--------- gnommo/models.py | 5 ++ gnommo/renderer.py | 17 ++++-- gnommo/transformer.py | 59 +++++++++++++----- tests/test_chunking_v2.py | 123 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 233 insertions(+), 64 deletions(-) create mode 100644 tests/test_chunking_v2.py diff --git a/docs/chunking_v2.md b/docs/chunking_v2.md index 235240d..5100874 100644 --- a/docs/chunking_v2.md +++ b/docs/chunking_v2.md @@ -1,6 +1,9 @@ # Chunked Rendering v2 — Design Spec -Status: **planned** (v1 shipped; v1 boundary limitation is detected + warned, not yet fixed) +Status: **implemented on branch `chunking-v2`, pending render-seam validation on the rig.** +Plan-level logic is covered by `tests/test_chunking_v2.py` (all green). What remains +is confirming the ffmpeg concat seam is frame/phase-accurate on a real render — see +"Concat-seam correctness" below. ## Why chunking exists @@ -113,16 +116,26 @@ via `skip_override`. Validation plan: ## Work items -- [ ] `_extract_video_events`: overlap test + `skip_override` (loop-aware). -- [ ] `_extract_audio_events`: add `end_time`/`skip_override`, overlap + seek. -- [ ] `AudioEvent`: `skip_override` field; renderer audio path honors it. -- [ ] `OutroEvent`: same treatment. -- [ ] Frame/audio seam-diff test (chunked vs full) in the test suite. -- [ ] Flip `_chunk_boundary_span_warnings` from "will be dropped" to a debug-only - assertion once v2 is the default. +- [x] `_extract_video_events`: overlap test + `skip_override` (loop-aware). +- [x] `_extract_audio_events`: overlap + `src_offset` seek (loop phase / linear). +- [x] `AudioEvent`: `src_offset` field; renderer audio paths (loop-with-pauses, + standard loop, one-shot) honor it. +- [x] `VideoEvent.skip_override`: renderer video input `-ss` honors it; the clip's + embedded audio (`tvaud`) is seeked automatically by the same input seek. +- [x] `_chunk_boundary_span_warnings`: downgraded from the "will be dropped" v1 + warning to an informational note (logged; terminal only under `--verbose`). +- [x] Plan-level tests: `tests/test_chunking_v2.py`. +- [ ] **Render-seam validation on the rig** (chunked-vs-full frame/audio diff) — the + remaining gate before making v2 the trusted default. +- [ ] Crossfade-loop audio (`_build_crossfade_loop_filter`) does not yet apply + `src_offset` — a crossfaded looping bed restarts phase at the seam. Standard + (non-crossfade) loops and one-shots are handled. Low priority. +- [ ] `OutroEvent`: not needed — outros are extracted for the last chunk only + (`config.outro if is_last_chunk`), so they never split across a seam. -## Seams already in place (v2 preparations, shipped) +## Not affected (verified) -- `VideoEvent.skip_override` (models.py) — inert; renderer honors it when set. -- `cli._chunk_boundary_span_warnings` — detection + warning. -- v1-limitation comments at both filter sites in `transformer.py`. +- Slides: `_extract_slide_events` already used overlap+clamp — no change. +- Full-screen `plan.background`: a separate always-included input. +- Full (non-chunked) render: `time_range=None` path leaves `skip_override`/ + `src_offset` at their defaults, so output is byte-identical to before. diff --git a/gnommo/cli.py b/gnommo/cli.py index 19a874e..737c25a 100644 --- a/gnommo/cli.py +++ b/gnommo/cli.py @@ -4137,16 +4137,14 @@ def _writeback_video_metadata(plan, project_path, config) -> None: def _chunk_boundary_span_warnings(plan, groups) -> list[str]: - """Detect events that span a chunk boundary. + """Report clips that span a chunk boundary. - KNOWN LIMITATION (chunking v1): the render's time-range filter keeps only events - whose *start* falls inside a chunk's window (transformer.py `_extract_video_events` - / `_extract_audio_events`: `if start_time < range_start: continue`). An overlay - video or audio clip that began in an earlier chunk and is still playing across the - boundary is therefore DROPPED from every later chunk it overlaps — so the chunked - output silently diverges from a full render. This detector makes that non-silent. - The real fix (overlap-inclusion + per-event seek) is chunking v2 — see - docs/chunking_v2.md. + Chunking v2 (docs/chunking_v2.md) INCLUDES these in the later chunk and seeks + into them (VideoEvent.skip_override / AudioEvent.src_offset) so they resume + mid-clip instead of being dropped. The seam then relies on `-c copy` joining + frame-aligned chunks, which is the one thing worth eyeballing — so this stays as + an informational list (logged; shown on the terminal only with --verbose), not + the hard "will be dropped" warning of v1. """ slide_start = {e.slide_id: e.start_time for e in plan.slide_events} boundaries = [] # (slide_id, output_time) at each chunk seam after the first @@ -4166,8 +4164,8 @@ def _chunk_boundary_span_warnings(plan, groups) -> list[str]: extra = f" (and {len(crossed) - 1} more)" if len(crossed) > 1 else "" warnings.append( f"{kind} '{label}' plays {_format_time(start)}–{_format_time(end)} and " - f"crosses the chunk boundary at {sid} ({_format_time(bt)}){extra}; it will " - f"be DROPPED from the chunk(s) after the boundary." + f"crosses the chunk boundary at {sid} ({_format_time(bt)}){extra}; v2 " + f"seeks into it so it continues across the seam." ) for e in plan.video_events: @@ -4211,29 +4209,23 @@ def _chunked_render( f"\n Auto-chunking: {len(slide_ids)} slides → {len(groups)} chunks of ≤{chunk_size}" ) - # Warn about events that span a chunk boundary — chunking v1 drops these from - # the later chunk, so the output would silently differ from a full render. + # Report clips that span a chunk boundary. v2 seeks into them so they continue + # across the seam (no longer dropped); this is informational — logged always, + # and echoed to the terminal only under --verbose — with a nudge to eyeball the + # seam since concat uses -c copy. if plan is not None: - _span_warnings = _chunk_boundary_span_warnings(plan, groups) - if _span_warnings: - banner = " " + "!" * 64 - print(f"\n{banner}", file=sys.stderr) - print(" CHUNK-BOUNDARY WARNING — chunked output will differ from a full render:", - file=sys.stderr) - for w in _span_warnings: - print(f" - {w}", file=sys.stderr) - print(" These overlay video/audio clips span a chunk seam. Chunking v1 keeps", - file=sys.stderr) - print(" only clips that START inside a chunk, so the ones above vanish after", - file=sys.stderr) - print(" the boundary. Options: render this project WITHOUT chunking (lower", - file=sys.stderr) - print(" cpu_limit instead), or move the chunk size so no seam splits them.", - file=sys.stderr) - print(f"{banner}\n", file=sys.stderr) - _render_log("CHUNK-BOUNDARY WARNING (v1 drops cross-boundary clips):") - for w in _span_warnings: + _span_notes = _chunk_boundary_span_warnings(plan, groups) + if _span_notes: + _render_log(f"chunking v2: {len(_span_notes)} clip(s) span a boundary (seam-seeked):") + for w in _span_notes: _render_log(f" - {w}") + if verbose: + print(f"\n {len(_span_notes)} clip(s) span a chunk boundary — v2 seeks across the seam:", + file=sys.stderr) + for w in _span_notes: + print(f" - {w}", file=sys.stderr) + print(" If a seam looks off, verify frame alignment (concat uses -c copy).", + file=sys.stderr) chunks_dir = out_dir / "chunks" chunks_dir.mkdir(parents=True, exist_ok=True) diff --git a/gnommo/models.py b/gnommo/models.py index ef389e2..c8d57e0 100644 --- a/gnommo/models.py +++ b/gnommo/models.py @@ -416,6 +416,11 @@ class AudioEvent: audio_id: str start_time: float # When to start playing (marker time - offset) audio_def: AudioDefinition + # Chunking v2 (docs/chunking_v2.md): when a clip began before this chunk's + # window it must resume mid-track, not restart at the seam. src_offset is the + # position (seconds) into the source stream to begin at — the loop phase for + # looping music, or a linear seek for one-shots. 0.0 = play from the start (v1). + src_offset: float = 0.0 @dataclass diff --git a/gnommo/renderer.py b/gnommo/renderer.py index 641eee4..844e44b 100644 --- a/gnommo/renderer.py +++ b/gnommo/renderer.py @@ -1473,7 +1473,9 @@ def build_filter_complex( for p in plan.narration_pauses if p.output_time > event.start_time ] - src_pos = 0.0 + # Chunking v2: start partway into the looped stream when the + # clip began in an earlier chunk (docs/chunking_v2.md). + src_pos = getattr(event, "src_offset", 0.0) seg_start = event.start_time seg_count = 0 @@ -1530,21 +1532,26 @@ def build_filter_complex( ) filters.extend(crossfade_filters) else: - # Standard loop without crossfade + # Standard loop without crossfade. Chunking v2: seek to + # the loop phase when the clip began in an earlier chunk. + _off = getattr(event, "src_offset", 0.0) filters.append( f"[{audio_idx}:a]aloop=loop=-1:size=2e+09," - f"atrim=0:{remaining:.3f}," + f"atrim={_off:.3f}:{_off + remaining:.3f}," f"asetpts=PTS-STARTPTS," f"adelay={delay_ms}|{delay_ms}," f"volume={volume:.2f}[{label}]" ) audio_labels_to_mix.append(f"[{label}]") else: - # One-shot audio: delay to trigger time + # One-shot audio: delay to trigger time. Chunking v2: seek in if + # the clip began in an earlier chunk (docs/chunking_v2.md). label = f"aud{i}" delay_ms = int(event.start_time * 1000) + _off = getattr(event, "src_offset", 0.0) + _seek = f"atrim={_off:.3f},asetpts=PTS-STARTPTS," if _off > 0 else "" filters.append( - f"[{audio_idx}:a]adelay={delay_ms}|{delay_ms},volume={volume:.2f}[{label}]" + f"[{audio_idx}:a]{_seek}adelay={delay_ms}|{delay_ms},volume={volume:.2f}[{label}]" ) audio_labels_to_mix.append(f"[{label}]") diff --git a/gnommo/transformer.py b/gnommo/transformer.py index a7fd37c..84d3b8d 100644 --- a/gnommo/transformer.py +++ b/gnommo/transformer.py @@ -1321,15 +1321,25 @@ def _extract_video_events( # end_on is None and marker_type == "narration": runs to end end_time = total_duration - # Filter by time range. - # CHUNKING v1 LIMITATION: this keeps only clips whose START is inside the - # window, so a clip that began in an earlier chunk and is still playing - # across the boundary is DROPPED here — the chunked output then differs from - # a full render. cli._chunk_boundary_span_warnings surfaces this. The v2 fix - # is overlap-inclusion + a per-event seek (skip_override); see - # docs/chunking_v2.md. - if start_time < range_start or start_time >= range_end: + # Filter by time range — CHUNKING v2 (docs/chunking_v2.md). + # Include any clip that OVERLAPS the window (not just those starting inside + # it), so a clip spanning a chunk boundary survives into the later chunk. + if end_time <= range_start or start_time >= range_end: continue + # A clip that began before this window is already mid-playback at the seam; + # seek into it so it resumes at the right frame instead of restarting. + skip_override = None + if start_time < range_start: + into = range_start - start_time # elapsed since the clip started + base = video_source.skip or 0.0 + playable = (video_source.duration - base) if video_source.duration else None + if playable and playable > 0 and into >= playable: + # the clip has looped by the window start → resume at the loop phase + skip_override = base + (into % playable) + else: + # still within the first play-through (or unknown length) → linear seek + skip_override = base + into + start_time = range_start # -> 0 after time_offset subtraction end_time = min(end_time, range_end) events.append( @@ -1341,6 +1351,7 @@ def _extract_video_events( cutout=cutout, cutout_name=cutout_name, layer=layer, + skip_override=skip_override, ) ) @@ -1367,17 +1378,35 @@ def _extract_audio_events( elif marker_id.startswith("audio:"): audio_id = marker_id[6:] if audio_id is not None and audio_id in audio: - # CHUNKING v1 LIMITATION (same as video, worse for looping background - # music): a clip started before the window is dropped from later chunks. - # v2 = overlap-inclusion + audio seek; see docs/chunking_v2.md. - if timing.timestamp < range_start or timing.timestamp >= range_end: + adef = audio[audio_id] + astart = max(0, timing.timestamp - AUDIO_OFFSET_SECONDS) + # Effective end of this clip on the output timeline. + if adef.loop: + aend = range_end # a loop fills to the window/render end + elif adef.duration is not None: + aend = astart + adef.duration + else: + aend = float("inf") # unknown one-shot length — assume it may span + # CHUNKING v2 (docs/chunking_v2.md): include if it OVERLAPS the window, + # and seek into clips that began earlier so they resume mid-track — the + # loop phase for looping music, a linear seek for one-shots. v1 dropped + # these, silencing looping background music in every chunk but the first. + if aend <= range_start or astart >= range_end: continue - start_time = max(0, timing.timestamp - AUDIO_OFFSET_SECONDS) + src_offset = 0.0 + if astart < range_start: + into = range_start - astart + if adef.loop and adef.duration: + src_offset = into % adef.duration + else: + src_offset = into + astart = range_start events.append( AudioEvent( audio_id=audio_id, - start_time=start_time, - audio_def=audio[audio_id], + start_time=astart, + audio_def=adef, + src_offset=src_offset, ) ) diff --git a/tests/test_chunking_v2.py b/tests/test_chunking_v2.py new file mode 100644 index 0000000..74fd744 --- /dev/null +++ b/tests/test_chunking_v2.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Plan-level validation for chunking v2 (docs/chunking_v2.md). + +Verifies the transformer now INCLUDES clips that span a chunk boundary and seeks +into them (skip_override / src_offset), instead of the v1 behaviour that dropped +them. This is a pure plan-level check — the ffmpeg concat-seam (frame alignment via +-c copy) still needs a real render on the rig to confirm. + +Run: ./venv/bin/python tests/test_chunking_v2.py +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from gnommo.transformer import ( + MarkerTiming, + _extract_audio_events, + _extract_video_events, + AUDIO_OFFSET_SECONDS, +) +from gnommo.models import AudioDefinition, VideoSource, CutoutDefinition, SlideDefinition + +_fails = [] + + +def check(name, cond, detail=""): + print(f" {'PASS' if cond else 'FAIL'} {name}" + (f" — {detail}" if detail and not cond else "")) + if not cond: + _fails.append(name) + + +# ── audio ──────────────────────────────────────────────────────────────────── +def test_audio(): + print("audio events:") + audio = { + "music": AudioDefinition(file="music.mp3", loop=True, duration=90.0), + "sfx": AudioDefinition(file="sfx.wav", loop=False, duration=500.0), + "blip": AudioDefinition(file="blip.wav", loop=False, duration=50.0), + } + # music triggers at t=0, sfx at t=10, blip at t=10 + markers = [ + MarkerTiming(marker_id="Amusic", timestamp=0.0, context="", confidence=1.0), + MarkerTiming(marker_id="Asfx", timestamp=10.0, context="", confidence=1.0), + MarkerTiming(marker_id="Ablip", timestamp=10.0, context="", confidence=1.0), + ] + + # Chunk window [300, 600): all three started earlier. + evs = {e.audio_id: e for e in _extract_audio_events(markers, audio, time_range=(300.0, 600.0))} + + # Looping music: included, resumes at loop phase. astart = max(0, 0-1)=0; into=300; + # phase = 300 % 90 = 30. + check("looping music spanning boundary is INCLUDED (v1 dropped it)", "music" in evs) + if "music" in evs: + m = evs["music"] + check("music clamped to window start", abs(m.start_time - 300.0) < 1e-6, f"start={m.start_time}") + check("music seeks to loop phase 30.0", abs(m.src_offset - 30.0) < 1e-6, f"src_offset={m.src_offset}") + + # One-shot still playing at the window: included, linear seek. + # astart = max(0,10-1)=9; aend=9+500=509 > 300 → spans. into=300-9=291. + check("one-shot still playing is INCLUDED", "sfx" in evs) + if "sfx" in evs: + s = evs["sfx"] + check("sfx linear seek 291.0", abs(s.src_offset - 291.0) < 1e-6, f"src_offset={s.src_offset}") + + # One-shot that ended before the window: excluded (aend=9+50=59 < 300). + check("one-shot ended before window is EXCLUDED", "blip" not in evs) + + # Full render (no range): everything from the start, no seek. + full = {e.audio_id: e for e in _extract_audio_events(markers, audio, time_range=None)} + check("full render includes music with no seek", "music" in full and full["music"].src_offset == 0.0) + + +# ── video ──────────────────────────────────────────────────────────────────── +def test_video(): + print("video events:") + cutouts = {"fullscreen": CutoutDefinition(x=0, y=0, height=1080, width=1920)} + videos = { + "bg": VideoSource( + source_file="bg.mp4", cutout="fullscreen", layer="below", + duration=90.0, skip=0.0, end_on="next_video", + ) + } + slides = {f"S{i}": SlideDefinition(image=f"S{i}.png", type="slide") for i in range(1, 11)} + markers = [MarkerTiming(marker_id=f"S{i}", timestamp=(i - 1) * 60.0, context="", confidence=1.0) + for i in range(1, 11)] + # Background overlay starts at slide-7 time (360) and, as the only video with + # end_on next_video, runs to total_duration (600). + markers.append(MarkerTiming(marker_id="vfm:bg", timestamp=360.0, context="", confidence=1.0)) + + total = 600.0 + # Chunk window that STARTS AFTER the video began: [420, 600) (slides 8-10). + evs, _ = _extract_video_events(markers, videos, cutouts, slides, total, time_range=(420.0, 600.0)) + bg = next((e for e in evs if e.video_id == "bg"), None) + + check("spanning background video is INCLUDED in the later chunk (v1 dropped it)", bg is not None) + if bg is not None: + check("bg clamped to window start", abs(bg.start_time - 420.0) < 1e-6, f"start={bg.start_time}") + # into = 420-360 = 60; playable = 90-0 = 90; 60 < 90 → linear seek 60. + check("bg linear seek 60.0 (first play-through)", abs((bg.skip_override or 0) - 60.0) < 1e-6, + f"skip_override={bg.skip_override}") + + # Window starting deep enough that the 90s clip has looped once: [480, 600). + # into = 480-360 = 120; 120 >= 90 → phase = 120 % 90 = 30. + evs2, _ = _extract_video_events(markers, videos, cutouts, slides, total, time_range=(480.0, 600.0)) + bg2 = next((e for e in evs2 if e.video_id == "bg"), None) + check("looped background resumes at phase 30.0", bg2 is not None and abs((bg2.skip_override or 0) - 30.0) < 1e-6, + f"skip_override={getattr(bg2, 'skip_override', None)}") + + # Full render: no seek. + evs3, _ = _extract_video_events(markers, videos, cutouts, slides, total, time_range=None) + bg3 = next((e for e in evs3 if e.video_id == "bg"), None) + check("full render includes bg with no seek", bg3 is not None and bg3.skip_override is None) + + +if __name__ == "__main__": + test_audio() + test_video() + print() + if _fails: + print(f"FAILED: {len(_fails)} check(s): {', '.join(_fails)}") + sys.exit(1) + print("All chunking-v2 plan-level checks passed.")