From 9824f6d08121ff0855cc77c8e83a50b264d4b78c Mon Sep 17 00:00:00 2001 From: jenstandstad Date: Sun, 26 Jul 2026 11:31:38 +0200 Subject: [PATCH] Adding memory guardrails --- gnommo/cli.py | 46 +++++++++++++++++++ gnommo/preprocessor.py | 100 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/gnommo/cli.py b/gnommo/cli.py index 46f766e..c3c8ed8 100644 --- a/gnommo/cli.py +++ b/gnommo/cli.py @@ -4525,6 +4525,51 @@ def _write_render_log_header(logfile, project_path, res, slides_arg, force, chun logfile.flush() +def _preflight_memory_advisory(plan, config, res: str) -> None: + """Warn BEFORE launching ffmpeg when the render is likely to exceed available + RAM, so an OOM kill is anticipated (with mitigations) rather than a surprise. + + ffmpeg opens every -i input up front and each decoder holds frame buffers, so + peak memory scales with the concurrent input count times the frame size. The + estimate is deliberately rough — it only fires when memory is genuinely tight, + so it stays quiet on the roomy render rig and speaks up on an 8 GB VM. + """ + mem = _system_mem() + if not mem or not mem[1]: + return + avail, _total = mem + n_inputs = ( + len(getattr(plan, "video_events", []) or []) + + len(getattr(plan, "outro_events", []) or []) + + len(getattr(plan, "narration_segments", []) or []) + + (1 if getattr(plan, "background", None) else 0) + ) + if n_inputs <= 0: + return + try: + w, h = config.resolution + except Exception: + w, h = 1920, 1080 + # ~0.15 GB per active 4K video input (decoder + swscale + filter buffers), + # scaled by output pixel count, plus a base for ffmpeg + python themselves. + per_input_gb = 0.15 * (w * h) / (3840 * 2160) + est_gb = 0.5 + n_inputs * per_input_gb + avail_gb = avail / 1e9 + _render_log( + f"[preflight] est ~{est_gb:.1f} GB for {n_inputs} inputs @ {w}x{h}, " + f"{avail_gb:.1f} GB free" + ) + if est_gb > 0.8 * avail_gb: + print( + f" ! Memory advisory: ~{est_gb:.1f} GB estimated for {n_inputs} inputs " + f"at {w}x{h}, but only {avail_gb:.1f} GB free — risk of an OOM kill." + ) + print( + " Consider: render --res low, a smaller chunk_slides, or freeing RAM. " + "(rough estimate; if it survives, ignore me)" + ) + + # Minimum ffmpeg the render is validated against. Older builds (e.g. Ubuntu 24.04's # 6.1.1) mis-mix the paused-narration audio — the talking head goes quiet before # interstitial videos and recovers after — silently producing a wrong file. @@ -5145,6 +5190,7 @@ def _cmd_render_impl( return 0 print("\n[4/4] Rendering...") + _preflight_memory_advisory(plan, config, res) # Record the exact ffmpeg command in the log only (not the terminal), so a # render that gets hard-killed mid-encode can still be reproduced/diagnosed. try: diff --git a/gnommo/preprocessor.py b/gnommo/preprocessor.py index f312fbc..83259fe 100644 --- a/gnommo/preprocessor.py +++ b/gnommo/preprocessor.py @@ -331,6 +331,103 @@ def set_ffmpeg_loglevel(level: Optional[str]) -> None: _FFMPEG_LOGLEVEL = level +def _mem_snapshot(): + """(available_bytes, total_bytes) of physical RAM, or None. Lightweight; used + only on the OOM error path so it can be duplicated from cli._system_mem without + a circular import.""" + try: + import psutil + vm = psutil.virtual_memory() + return vm.available, vm.total + except Exception: + pass + if sys.platform.startswith("linux"): + try: + info = {} + with open("/proc/meminfo") as f: + for line in f: + k, _, v = line.partition(":") + info[k.strip()] = int(v.strip().split()[0]) * 1024 + return info.get("MemAvailable", info.get("MemFree", 0)), info.get("MemTotal", 0) + except Exception: + return None + return None + + +def _kernel_oom_lines(): + """Best-effort: the kernel's own OOM-killer log lines (Linux). Empty when not + Linux or when dmesg is restricted (dmesg_restrict=1 without privileges) — in + that case the signal/memory evidence still tells the story.""" + if not sys.platform.startswith("linux"): + return [] + for probe in ( + ["dmesg", "--ctime"], + ["dmesg"], + ["journalctl", "-k", "--no-pager", "-n", "300"], + ): + try: + r = subprocess.run(probe, capture_output=True, text=True, timeout=5) + except Exception: + continue + if r.returncode != 0 or not r.stdout: + continue + hits = [ + ln.strip()[:200] + for ln in r.stdout.splitlines() + if any(s in ln.lower() for s in ("out of memory", "oom-kill", "killed process")) + ] + if hits: + return hits[-4:] + return [] + + +def _oom_postmortem(returncode: int, log_text: str) -> None: + """When ffmpeg dies from memory pressure, print a diagnosis instead of a bare + exit code. Detects the OS OOM killer's signature (SIGKILL) and explicit + allocation failures, reports current RAM, pulls the kernel oom-kill line when + readable, and lists mitigations. No-op for ordinary (non-OOM) failures.""" + low = (log_text or "").lower() + killed = returncode in (-9, 137) # SIGKILL, or 128+9 via a shell wrapper + alloc_fail = ( + "cannot allocate memory" in low + or "out of memory" in low + or "error while allocating" in low + or (returncode in (-6, 134) and "memory" in low) + ) + if not (killed or alloc_fail): + return + + out = ["", " " + "=" * 66, + " OOM DIAGNOSIS — ffmpeg was killed by memory pressure, not a normal error."] + if killed: + out += [ + " Signal: SIGKILL (-9) — the signature of the OS out-of-memory killer.", + " The kernel terminated ffmpeg instantly, so ffmpeg logged no error of its", + " own; that is why the only visible symptom was 'process terminated'.", + ] + if alloc_fail: + out.append(" ffmpeg reported an allocation failure (could not allocate memory).") + mem = _mem_snapshot() + if mem and mem[1]: + avail, total = mem + out.append( + f" Memory now: {avail / 1e9:.2f} GB free of {total / 1e9:.1f} GB " + f"({100 * (total - avail) / total:.0f}% used)." + ) + for kline in _kernel_oom_lines(): + out.append(f" kernel: {kline}") + out += [ + " Mitigations (most effective first):", + " - render --res low smaller frames -> much less RAM per input", + " - render in slide chunks: raise chunk_slides / render_chunk_slides", + " - lower ffmpeg thread count in ~/.gnommo.conf (fewer parallel buffers)", + " - add RAM or swap; on WSL2 raise memory=/swap= in .wslconfig", + " " + "=" * 66, ""] + msg = "\n".join(out) + sys.stdout.write(msg + "\n") + sys.stdout.flush() + + def run_ffmpeg_with_progress(cmd, duration, description="Processing", loglevel=None): from collections import deque @@ -364,6 +461,8 @@ def run_ffmpeg_with_progress(cmd, duration, description="Processing", loglevel=N sys.stdout.write(line) sys.stdout.flush() p.wait() + if p.returncode != 0: + _oom_postmortem(p.returncode, "".join(logs)) return subprocess.CompletedProcess(cmd, p.returncode, stdout="", stderr="".join(logs)) insert_pos = cmd.index("-y") + 1 if "-y" in cmd else 1 @@ -468,6 +567,7 @@ def run_ffmpeg_with_progress(cmd, duration, description="Processing", loglevel=N ) sys.stdout.write(f"\n FFmpeg exited with code {code}{signal_hint}\n") sys.stdout.flush() + _oom_postmortem(code, "".join(logs)) return subprocess.CompletedProcess( cmd, p.returncode, stdout="", stderr="".join(logs)