Adding memory awareness to the render
This commit is contained in:
+163
-18
@@ -4279,6 +4279,146 @@ def _render_log(msg: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _system_mem():
|
||||
"""Return (available_bytes, total_bytes) of physical RAM, or None.
|
||||
|
||||
Cross-platform without requiring psutil (the machine that OOM-crashes is
|
||||
Windows): prefers psutil, falls back to Windows GlobalMemoryStatusEx and
|
||||
Linux /proc/meminfo so system-memory pressure is visible everywhere.
|
||||
"""
|
||||
try:
|
||||
import psutil
|
||||
|
||||
vm = psutil.virtual_memory()
|
||||
return vm.available, vm.total
|
||||
except Exception:
|
||||
pass
|
||||
if sys.platform.startswith("win"):
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
class _MEMSTAT(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("dwLength", ctypes.c_ulong),
|
||||
("dwMemoryLoad", ctypes.c_ulong),
|
||||
("ullTotalPhys", ctypes.c_ulonglong),
|
||||
("ullAvailPhys", ctypes.c_ulonglong),
|
||||
("ullTotalPageFile", ctypes.c_ulonglong),
|
||||
("ullAvailPageFile", ctypes.c_ulonglong),
|
||||
("ullTotalVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailExtendedVirtual", ctypes.c_ulonglong),
|
||||
]
|
||||
|
||||
m = _MEMSTAT()
|
||||
m.dwLength = ctypes.sizeof(_MEMSTAT)
|
||||
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(m))
|
||||
return int(m.ullAvailPhys), int(m.ullTotalPhys)
|
||||
except Exception:
|
||||
return None
|
||||
if sys.platform.startswith("linux"):
|
||||
try:
|
||||
info = {}
|
||||
with open("/proc/meminfo") as f:
|
||||
for line in f:
|
||||
k, _, v = line.partition(":")
|
||||
info[k.strip()] = int(v.strip().split()[0]) * 1024
|
||||
return info.get("MemAvailable", info.get("MemFree", 0)), info.get("MemTotal", 0)
|
||||
except Exception:
|
||||
return None
|
||||
if sys.platform == "darwin":
|
||||
try:
|
||||
import os as _os
|
||||
|
||||
page = _os.sysconf("SC_PAGE_SIZE")
|
||||
total = _os.sysconf("SC_PHYS_PAGES") * page
|
||||
out = subprocess.run(["vm_stat"], capture_output=True, text=True).stdout
|
||||
free = inactive = spec = 0
|
||||
for line in out.splitlines():
|
||||
num = line.split(":")[-1].strip().rstrip(".")
|
||||
if line.startswith("Pages free:"):
|
||||
free = int(num)
|
||||
elif line.startswith("Pages inactive:"):
|
||||
inactive = int(num)
|
||||
elif line.startswith("Pages speculative:"):
|
||||
spec = int(num)
|
||||
return (free + inactive + spec) * page, total
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _start_memory_sampler(logfile, interval: float = 3.0):
|
||||
"""Log memory every `interval`s in a background thread → OOM visibility.
|
||||
|
||||
Records system free/used RAM (always) and the gnommo+ffmpeg process-tree RSS
|
||||
(when psutil is available). Returns a stop callback + peak dict, or None. On a
|
||||
hard OOM kill the finally block never runs, but the per-sample lines are
|
||||
flushed as they happen, so the log shows memory climbing right up to the kill.
|
||||
"""
|
||||
import threading
|
||||
|
||||
total = _system_mem()
|
||||
if total is None:
|
||||
_render_log("memory sampler: unavailable on this platform")
|
||||
return None
|
||||
|
||||
try:
|
||||
import psutil
|
||||
|
||||
_proc = psutil.Process()
|
||||
except Exception:
|
||||
psutil = None
|
||||
_proc = None
|
||||
_render_log(
|
||||
"memory sampler: system RAM only ('pip install psutil' adds per-process RSS)"
|
||||
)
|
||||
|
||||
stop = threading.Event()
|
||||
peak = {"rss": 0, "used_pct": 0.0}
|
||||
|
||||
def _tree_rss():
|
||||
if _proc is None:
|
||||
return None
|
||||
try:
|
||||
rss = _proc.memory_info().rss
|
||||
for c in _proc.children(recursive=True):
|
||||
try:
|
||||
rss += c.memory_info().rss
|
||||
except Exception:
|
||||
pass
|
||||
return rss
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _loop():
|
||||
while not stop.wait(interval):
|
||||
mem = _system_mem()
|
||||
if not mem:
|
||||
continue
|
||||
avail, tot = mem
|
||||
used_pct = 100.0 * (tot - avail) / tot if tot else 0.0
|
||||
peak["used_pct"] = max(peak["used_pct"], used_pct)
|
||||
rss = _tree_rss()
|
||||
if rss is not None:
|
||||
peak["rss"] = max(peak["rss"], rss)
|
||||
rss_str = f"gnommo+ffmpeg={rss / 1e9:.2f}GB | "
|
||||
else:
|
||||
rss_str = ""
|
||||
try:
|
||||
logfile.write(
|
||||
f"[mem {datetime.now().strftime('%H:%M:%S')}] {rss_str}"
|
||||
f"system {used_pct:.0f}% used, {avail / 1e9:.2f}GB free of {tot / 1e9:.1f}GB\n"
|
||||
)
|
||||
logfile.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
t = threading.Thread(target=_loop, daemon=True)
|
||||
t.start()
|
||||
return stop, peak
|
||||
|
||||
|
||||
def _write_render_log_header(logfile, project_path, res, slides_arg, force, chunk_slides):
|
||||
import os
|
||||
import platform as _platform
|
||||
@@ -4289,13 +4429,12 @@ def _write_render_log_header(logfile, project_path, res, slides_arg, force, chun
|
||||
except Exception:
|
||||
ffmpeg_ver = "unavailable"
|
||||
|
||||
try:
|
||||
import psutil # optional dependency
|
||||
|
||||
vm = psutil.virtual_memory()
|
||||
mem = f"{vm.available / 1e9:.1f} GB free / {vm.total / 1e9:.1f} GB total"
|
||||
except Exception:
|
||||
mem = "unknown (install psutil for memory info)"
|
||||
_m = _system_mem()
|
||||
mem = (
|
||||
f"{_m[0] / 1e9:.1f} GB free / {_m[1] / 1e9:.1f} GB total"
|
||||
if _m
|
||||
else "unknown"
|
||||
)
|
||||
|
||||
logfile.write("=" * 70 + "\n")
|
||||
logfile.write(f"gnommo render log — {project_path.name}\n")
|
||||
@@ -4355,6 +4494,7 @@ def cmd_render(
|
||||
sys.stdout = _TeeStream(_orig_out, logfile)
|
||||
sys.stderr = _TeeStream(_orig_err, logfile)
|
||||
_RENDER_LOGFILE = logfile
|
||||
_sampler = _start_memory_sampler(logfile)
|
||||
try:
|
||||
return _cmd_render_impl(project_path, verbose, dry_run, **passthrough)
|
||||
except BaseException:
|
||||
@@ -4365,6 +4505,13 @@ def cmd_render(
|
||||
logfile.flush()
|
||||
raise
|
||||
finally:
|
||||
if _sampler is not None:
|
||||
_stop, _peak = _sampler
|
||||
_stop.set()
|
||||
_peak_rss = f"{_peak['rss'] / 1e9:.2f} GB tree RSS, " if _peak["rss"] else ""
|
||||
logfile.write(
|
||||
f"\n[peak memory] {_peak_rss}system peaked at {_peak['used_pct']:.0f}% used\n"
|
||||
)
|
||||
sys.stdout = _orig_out
|
||||
sys.stderr = _orig_err
|
||||
_RENDER_LOGFILE = None
|
||||
@@ -4710,18 +4857,16 @@ def _cmd_render_impl(
|
||||
output_path = _output_path_override
|
||||
out_dir = output_path.parent
|
||||
out_filename = output_path.name
|
||||
elif config.output_video:
|
||||
out_filename = config.output_video
|
||||
out_dir = project_path / "out" / res if res != "full" else project_path / "out"
|
||||
output_path = out_dir / out_filename
|
||||
elif slide_range:
|
||||
start, end = slide_range
|
||||
range_suffix = f"_{start}-{end}" if end else f"_{start}-end"
|
||||
out_filename = f"final{range_suffix}.mp4"
|
||||
out_dir = project_path / "out" / res if res != "full" else project_path / "out"
|
||||
output_path = out_dir / out_filename
|
||||
else:
|
||||
out_filename = f"{config.co}.mp4"
|
||||
base = config.output_video if config.output_video else f"{config.co}.mp4"
|
||||
# A partial (--slides) render appends the range to the filename so that
|
||||
# e.g. S1:S9 and S10:S19 don't overwrite each other (or the full render).
|
||||
if slide_range:
|
||||
start, end = slide_range
|
||||
rng = f"{start}_{end}" if end else f"{start}_end"
|
||||
base_p = Path(base)
|
||||
base = f"{base_p.stem}_{rng}{base_p.suffix or '.mp4'}"
|
||||
out_filename = base
|
||||
out_dir = project_path / "out" / res if res != "full" else project_path / "out"
|
||||
output_path = out_dir / out_filename
|
||||
|
||||
|
||||
Reference in New Issue
Block a user