From 46e243b4dc476667a7c807026ccec0bf09795a38 Mon Sep 17 00:00:00 2001 From: jenstandstad Date: Sun, 26 Jul 2026 11:20:34 +0200 Subject: [PATCH] Fixing import stage --- gnommo/cli.py | 60 ++++++++-- gnommo/transfer.py | 272 +++++++-------------------------------------- 2 files changed, 87 insertions(+), 245 deletions(-) diff --git a/gnommo/cli.py b/gnommo/cli.py index c8e15c7..46f766e 100644 --- a/gnommo/cli.py +++ b/gnommo/cli.py @@ -853,6 +853,34 @@ def _import_shared_audio( print(f" No new shared audio files to add") +def _probe_is_fresh(cached: dict, path: Path, *fields: str) -> bool: + """True when `cached` already holds every field in `fields` AND was probed from + the file as it exists now (its stored ``src_mtime`` still matches the file on + disk). + + A file re-exported after import gets a newer mtime, so its entry re-probes + automatically on the next import — no --force needed. This is what stops a stale + ``has_audio``/``duration`` (e.g. a render clip regenerated without an audio + track) from silently persisting and later crashing the render with + "[N:a] matches no streams". Entries written before mtime-stamping existed have no + ``src_mtime`` and re-probe once to gain the stamp. + """ + if not all(f in cached for f in fields): + return False + stamp = cached.get("src_mtime") + if stamp is None: + return False + try: + return abs(float(stamp) - path.stat().st_mtime) < 1.0 + except OSError: + return False + + +def _file_mtime(path: Path) -> float: + """Source-file mtime to stamp into a probed entry (rounded for stable JSON).""" + return round(path.stat().st_mtime, 3) + + def _probe_audio_durations( project_path: Path, config, @@ -882,10 +910,6 @@ def _probe_audio_durations( for audio_id, audio_data in data.items(): if "file" not in audio_data: continue - if "duration" in audio_data and not force: - if verbose: - print(f" Audio '{audio_id}': cached ({audio_data['duration']:.1f}s)") - continue if audio_data.get("is_shared") and shared_assets_dir: audio_path = shared_assets_dir / "media" / "audio" / audio_data["file"] else: @@ -894,12 +918,17 @@ def _probe_audio_durations( if verbose: print(f" Audio '{audio_id}': file not found, skipping") continue + if not force and _probe_is_fresh(audio_data, audio_path, "duration"): + if verbose: + print(f" Audio '{audio_id}': cached ({audio_data['duration']:.1f}s)") + continue print( f" Probing audio '{audio_id}' ({audio_path.name})...", end=" ", flush=True ) try: duration = _get_audio_duration(audio_path) data[audio_id]["duration"] = round(duration, 3) + data[audio_id]["src_mtime"] = _file_mtime(audio_path) updated = True print(f"{duration:.1f}s") except Exception as e: @@ -962,13 +991,6 @@ def _probe_video_metadata( else: canonical = video_data - if not force and "duration" in canonical and "has_audio" in canonical: - if verbose: - print( - f" Video '{video_id}': cached ({canonical['duration']:.1f}s, audio={canonical['has_audio']})" - ) - continue - base_dir = ( shared_assets_dir if (is_shared and shared_assets_dir) else videos_dir ) @@ -1003,13 +1025,27 @@ def _probe_video_metadata( print(f" Video '{video_id}': file not found, skipping") continue + # Re-probe when the file has changed since the cached values were written + # (mtime mismatch) — a clip re-exported after import self-heals instead of + # carrying a stale has_audio into the render. + if not force and _probe_is_fresh(canonical, video_path, "duration", "has_audio"): + if verbose: + print( + f" Video '{video_id}': cached ({canonical['duration']:.1f}s, audio={canonical['has_audio']})" + ) + continue + print( f" Probing video '{video_id}' ({video_path.name})...", end=" ", flush=True ) try: duration = get_video_duration(video_path) has_audio = _has_audio_stream(video_path) - result = {"duration": round(duration, 3), "has_audio": has_audio} + result = { + "duration": round(duration, 3), + "has_audio": has_audio, + "src_mtime": _file_mtime(video_path), + } print(f"{duration:.1f}s, audio={has_audio}") if is_shared and video_id in shared_data: diff --git a/gnommo/transfer.py b/gnommo/transfer.py index b92e084..afa88c2 100644 --- a/gnommo/transfer.py +++ b/gnommo/transfer.py @@ -43,6 +43,8 @@ _COMMITS_LOG = "commits.log" _SYNC_EXCLUDES = [ "out/", "media/narration/processed/", + "media/narration/old/", + "media/narration/raw_mov/old/", "media/narration/intermediate/", "media/videos/intermediate/", "media/narration/low/", @@ -56,139 +58,7 @@ _SYNC_EXCLUDES = [ # --------------------------------------------------------------------------- -# Manifest builder -# --------------------------------------------------------------------------- - -def _build_manifest(project_path: Path) -> list[str]: - """Return sorted list of project-relative paths required to render.""" - files: set[str] = set() - - # Read project.json for path overrides - proj_json = project_path / "project.json" - project: dict = {} - if proj_json.exists(): - try: - project = json.loads(proj_json.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - pass - - # Fixed project-root files. events.json / scaffold.json are the precomputed - # timing layer the render needs — events.json also carries the human `adjustment` - # tweaks — so they must travel UP to the rig. (They're excluded on `down` so the - # rig's regenerated copies can't clobber the local edits.) - for fixed in [ - "project.json", - "manuscript.txt", - _COMMITS_LOG, - "citations.json", - "events.json", - "scaffold.json", - ]: - if (project_path / fixed).exists(): - files.add(fixed) - - # Keynote presentation(s) - for key_file in project_path.glob("*.key"): - files.add(key_file.name) - - # NOTE: out/ (rendered mp4/srt) is deliberately NOT pushed. The rendering - # rig produces those; pushing the local (older) copy up would overwrite the - # rig's fresh render, which 'down' would then pull back — clobbering the new - # result. Rendered output flows one way only: rig → local via 'down'. - - # Manuscript (may be at a non-standard path) - manuscript_rel = project.get("manuscript", "manuscript.txt") - if (project_path / manuscript_rel).exists(): - files.add(manuscript_rel) - - # Slide images + slides.json. - # The renderer reads this path lower-cased (parse_slides uses - # config.slides_path.lower()), because import may embed a capital-cased - # project name (e.g. "media/slides/Video3/" while the dir is "video3/"). - # Push to the SAME lower-cased location — otherwise on a case-sensitive - # server 'up' writes media/slides/Video3/slides.json while render reads - # media/slides/video3/slides.json and never sees the update. - slides_rel = project.get("slides", "media/slides").lower() - slides_path = project_path / slides_rel - # slides might point directly to slides.json — include its parent dir - if slides_path.is_file(): - files.add(slides_rel) - slides_path = slides_path.parent - if slides_path.is_dir(): - for f in slides_path.rglob("*"): - if f.is_file(): - files.add(str(f.relative_to(project_path))) - - # Narration - # Push narration.json (preserves per-segment trim/channel settings) plus the - # RAW recordings in raw_mov/. Preprocessing runs on the rig — it produces the - # large processed/*_processed.mov segments there. We deliberately do NOT push - # each entry's source_file: after a local preprocess that points at - # processed/..._processed.mov, and uploading those would both waste bandwidth - # on files the rig regenerates and let a stale local render input clobber the - # rig's freshly produced one. - narration_rel = project.get("narration", "media/narration/narration.json") - narration_json = project_path / narration_rel - narration_dir = narration_json.parent - if narration_json.exists(): - files.add(narration_rel) - raw_mov_dir = narration_dir / "raw_mov" - if raw_mov_dir.is_dir(): - for f in sorted(raw_mov_dir.glob("*.mov")): - if f.is_file() and not f.name.startswith("."): - files.add(str(f.relative_to(project_path))) - # Per-segment Whisper transcripts. These are the ONLY non-deterministic input - # to marker alignment (Whisper is not bit-reproducible across platforms), so - # syncing them makes `build` produce identical events.json on any machine — the - # rig can no longer diverge by re-transcribing locally. - transcripts_dir = narration_dir / "transcripts" - if transcripts_dir.is_dir(): - for f in sorted(transcripts_dir.glob("*.json")): - if f.is_file() and not f.name.startswith("."): - files.add(str(f.relative_to(project_path))) - - # Audio — standard location, non-shared entries only - audio_json = project_path / "media" / "audio" / "audio.json" - if audio_json.exists(): - files.add("media/audio/audio.json") - try: - data = json.loads(audio_json.read_text(encoding="utf-8")) - audio_dir = audio_json.parent - for entry in data.values(): - if entry.get("is_shared"): - continue - src = entry.get("file") - if src: - candidate = audio_dir / src - if candidate.exists(): - files.add(str(candidate.relative_to(project_path))) - except (json.JSONDecodeError, OSError): - pass - - # Videos — non-shared entries only - videos_rel = project.get("videos", "media/videos/videos.json") - videos_json = project_path / videos_rel - videos_dir = videos_json.parent - if videos_json.exists(): - files.add(videos_rel) - try: - data = json.loads(videos_json.read_text(encoding="utf-8")) - for entry in data.values(): - if entry.get("is_shared"): - continue - src = entry.get("source_file") - if src: - candidate = videos_dir / src - if candidate.exists(): - files.add(str(candidate.relative_to(project_path))) - except (json.JSONDecodeError, OSError): - pass - - return sorted(files) - - -# --------------------------------------------------------------------------- -# Shared assets manifest +# Shared assets # --------------------------------------------------------------------------- def _find_shared_assets_root(project_path: Path) -> Optional[Path]: @@ -208,69 +78,6 @@ def _find_shared_assets_root(project_path: Path) -> Optional[Path]: return None -def _build_shared_manifest(project_path: Path) -> tuple: - """Return (shared_assets_root, sorted list of root-relative paths) for shared files used by this project. - - Layout conventions discovered from the actual filesystem: - - Shared videos: shared_assets/{source_file} (e.g. Logo6sec.mov, pexels/14923961.mp4) - - Shared audio: shared_assets/media/audio/{file} - """ - shared_root = _find_shared_assets_root(project_path) - if shared_root is None: - return None, [] - - files: set[str] = set() - - # shared_assets/videos.json so the render pipeline can read metadata - if (shared_root / "videos.json").exists(): - files.add("videos.json") - - project: dict = {} - proj_json = project_path / "project.json" - if proj_json.exists(): - try: - project = json.loads(proj_json.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - pass - - # Shared video source files - videos_rel = project.get("videos", "media/videos/videos.json") - videos_json = project_path / videos_rel - if videos_json.exists(): - try: - data = json.loads(videos_json.read_text(encoding="utf-8")) - for entry in data.values(): - if not entry.get("is_shared"): - continue - src = entry.get("source_file") - if not src: - continue - # source_file is relative to shared_assets root - if (shared_root / src).exists(): - files.add(src) - except (json.JSONDecodeError, OSError): - pass - - # Shared audio source files (live under media/audio/ within shared_assets) - audio_json = project_path / "media" / "audio" / "audio.json" - if audio_json.exists(): - try: - data = json.loads(audio_json.read_text(encoding="utf-8")) - for entry in data.values(): - if not entry.get("is_shared"): - continue - src = entry.get("file") - if not src: - continue - rel = f"media/audio/{src}" - if (shared_root / rel).exists(): - files.add(rel) - except (json.JSONDecodeError, OSError): - pass - - return shared_root, sorted(files) - - # --------------------------------------------------------------------------- # Commit log helpers # --------------------------------------------------------------------------- @@ -335,59 +142,58 @@ def cmd_up(project_path: Path, verbose: bool, dry_run: bool) -> int: print(f" Run: gnommo -p {project_path.name} down") return 1 - # Build manifest - manifest = _build_manifest(project_path) - total = len(manifest) - print(f"Pushing: {project_path.name} ({total} files, last commit: {local_last})") - if verbose: - for f in manifest: - print(f" {f}") + print(f"Pushing: {project_path.name} (whole tree minus excludes, last commit: {local_last})") - # Build shared assets manifest - shared_root, shared_files = _build_shared_manifest(project_path) + shared_root = _find_shared_assets_root(project_path) remote_shared = f"{server['path']}/shared_assets" - if dry_run: - print("\n[DRY RUN] Would push project files:") - for f in manifest: - print(f" {f}") - if shared_files: - print(f"\n[DRY RUN] Would push shared assets ({len(shared_files)} files):") - for f in shared_files: - print(f" {f}") - return 0 - - # Pass 1: project files - subprocess.run([*ssh_cmd, f"mkdir -p {remote_project}"], check=True) + # Pass 1: project files — whole tree, denylist excludes. rsync_cmd = [ "rsync", "-av", "--progress", - "--files-from=-", "-e", f"ssh -p {server['port']}", + *[f"--exclude={p}" for p in _SYNC_EXCLUDES], f"{project_path}/", f"{server['user']}@{server['host']}:{remote_project}/", ] - result = subprocess.run(rsync_cmd, input="\n".join(manifest), text=True) + # Pass 2: shared assets — whole tree, same excludes. + rsync_shared = [ + "rsync", "-av", "--progress", + "-e", f"ssh -p {server['port']}", + *[f"--exclude={p}" for p in _SYNC_EXCLUDES], + f"{shared_root}/" if shared_root else "", + f"{server['user']}@{server['host']}:{remote_shared}/", + ] + + if dry_run: + dry_project = rsync_cmd[:1] + ["--dry-run"] + rsync_cmd[1:] + print("\n[DRY RUN] Would push project:") + print(f" {' '.join(dry_project)}") + if shared_root: + dry_shared = rsync_shared[:1] + ["--dry-run"] + rsync_shared[1:] + print("\n[DRY RUN] Would push shared assets:") + print(f" {' '.join(dry_shared)}") + return 0 + + if verbose: + print(f" {' '.join(rsync_cmd)}") + + # Pass 1: project files + subprocess.run([*ssh_cmd, f"mkdir -p {remote_project}"], check=True) + result = subprocess.run(rsync_cmd) if result.returncode != 0: print(f"Error: rsync failed for project files (code {result.returncode})") return 1 - # Pass 2: shared assets referenced by this project - if shared_files and shared_root: - print(f"\nPushing shared assets ({len(shared_files)} files)...") + # Pass 2: shared assets + if shared_root: + print("\nPushing shared assets...") subprocess.run([*ssh_cmd, f"mkdir -p {remote_shared}"], check=True) - rsync_shared = [ - "rsync", "-av", "--progress", - "--files-from=-", - "-e", f"ssh -p {server['port']}", - f"{shared_root}/", - f"{server['user']}@{server['host']}:{remote_shared}/", - ] - result = subprocess.run(rsync_shared, input="\n".join(shared_files), text=True) + result = subprocess.run(rsync_shared) if result.returncode != 0: print(f"Error: rsync failed for shared assets (code {result.returncode})") return 1 - print(f"\nDone. Pushed {total} project files, {len(shared_files)} shared assets.") + print("\nDone.") return 0 @@ -425,7 +231,7 @@ def cmd_down(project_path: Path, verbose: bool, dry_run: bool) -> int: rsync_cmd = [ "rsync", "-av", "--progress", "-e", f"ssh -p {server['port']}", - *[f"--exclude={p}" for p in _DOWN_EXCLUDES], + *[f"--exclude={p}" for p in _SYNC_EXCLUDES], f"{server['user']}@{server['host']}:{remote_project}/", f"{project_path}/", ] @@ -435,7 +241,7 @@ def cmd_down(project_path: Path, verbose: bool, dry_run: bool) -> int: rsync_shared_cmd = [ "rsync", "-av", "--progress", "-e", f"ssh -p {server['port']}", - *[f"--exclude={p}" for p in _DOWN_EXCLUDES], + *[f"--exclude={p}" for p in _SYNC_EXCLUDES], f"{server['user']}@{server['host']}:{remote_shared}/", f"{local_shared}/", ]