gitprov/gnommo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixing import stage

Browse Source
This commit is contained in:
gitprov
2026-07-26 11:20:34 +02:00
parent f79a0cb082
commit 46e243b4dc
2 changed files with 87 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gnommo/cli.py
+48 -12
View File
@@ -853,6 +853,34 @@ def _import_shared_audio(
print(f" No new shared audio files to add")
def _probe_is_fresh(cached: dict, path: Path, *fields: str) -> bool:
"""True when `cached` already holds every field in `fields` AND was probed from
the file as it exists now (its stored ``src_mtime`` still matches the file on
disk).
A file re-exported after import gets a newer mtime, so its entry re-probes
automatically on the next import — no --force needed. This is what stops a stale
``has_audio``/``duration`` (e.g. a render clip regenerated without an audio
track) from silently persisting and later crashing the render with
"[N:a] matches no streams". Entries written before mtime-stamping existed have no
``src_mtime`` and re-probe once to gain the stamp.
"""
if not all(f in cached for f in fields):
return False
stamp = cached.get("src_mtime")
if stamp is None:
return False
try:
return abs(float(stamp) - path.stat().st_mtime) < 1.0
except OSError:
return False
def _file_mtime(path: Path) -> float:
"""Source-file mtime to stamp into a probed entry (rounded for stable JSON)."""
return round(path.stat().st_mtime, 3)
def _probe_audio_durations(
project_path: Path,
config,
@@ -882,10 +910,6 @@ def _probe_audio_durations(
for audio_id, audio_data in data.items():
if "file" not in audio_data:
continue
if "duration" in audio_data and not force:
if verbose:
print(f" Audio '{audio_id}': cached ({audio_data['duration']:.1f}s)")
continue
if audio_data.get("is_shared") and shared_assets_dir:
audio_path = shared_assets_dir / "media" / "audio" / audio_data["file"]
else:
@@ -894,12 +918,17 @@ def _probe_audio_durations(
if verbose:
print(f" Audio '{audio_id}': file not found, skipping")
continue
if not force and _probe_is_fresh(audio_data, audio_path, "duration"):
if verbose:
print(f" Audio '{audio_id}': cached ({audio_data['duration']:.1f}s)")
continue
print(
f" Probing audio '{audio_id}' ({audio_path.name})...", end=" ", flush=True
)
try:
duration = _get_audio_duration(audio_path)
data[audio_id]["duration"] = round(duration, 3)
data[audio_id]["src_mtime"] = _file_mtime(audio_path)
updated = True
print(f"{duration:.1f}s")
except Exception as e:
@@ -962,13 +991,6 @@ def _probe_video_metadata(
else:
canonical = video_data
if not force and "duration" in canonical and "has_audio" in canonical:
if verbose:
print(
f" Video '{video_id}': cached ({canonical['duration']:.1f}s, audio={canonical['has_audio']})"
)
continue
base_dir = (
shared_assets_dir if (is_shared and shared_assets_dir) else videos_dir
)
@@ -1003,13 +1025,27 @@ def _probe_video_metadata(
print(f" Video '{video_id}': file not found, skipping")
continue
# Re-probe when the file has changed since the cached values were written
# (mtime mismatch) — a clip re-exported after import self-heals instead of
# carrying a stale has_audio into the render.
if not force and _probe_is_fresh(canonical, video_path, "duration", "has_audio"):
if verbose:
print(
f" Video '{video_id}': cached ({canonical['duration']:.1f}s, audio={canonical['has_audio']})"
)
continue
print(
f" Probing video '{video_id}' ({video_path.name})...", end=" ", flush=True
)
try:
duration = get_video_duration(video_path)
has_audio = _has_audio_stream(video_path)
result = {"duration": round(duration, 3), "has_audio": has_audio}
result = {
"duration": round(duration, 3),
"has_audio": has_audio,
"src_mtime": _file_mtime(video_path),
}
print(f"{duration:.1f}s, audio={has_audio}")
if is_shared and video_id in shared_data:
gnommo/transfer.py
+39 -233
View File
@@ -43,6 +43,8 @@ _COMMITS_LOG = "commits.log"
_SYNC_EXCLUDES = [
"out/",
"media/narration/processed/",
"media/narration/old/",
"media/narration/raw_mov/old/",
"media/narration/intermediate/",
"media/videos/intermediate/",
"media/narration/low/",
@@ -56,139 +58,7 @@ _SYNC_EXCLUDES = [
# ---------------------------------------------------------------------------
# Manifest builder
# ---------------------------------------------------------------------------
def _build_manifest(project_path: Path) -> list[str]:
"""Return sorted list of project-relative paths required to render."""
files: set[str] = set()
# Read project.json for path overrides
proj_json = project_path / "project.json"
project: dict = {}
if proj_json.exists():
try:
project = json.loads(proj_json.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
pass
# Fixed project-root files. events.json / scaffold.json are the precomputed
# timing layer the render needs — events.json also carries the human `adjustment`
# tweaks — so they must travel UP to the rig. (They're excluded on `down` so the
# rig's regenerated copies can't clobber the local edits.)
for fixed in [
"project.json",
"manuscript.txt",
_COMMITS_LOG,
"citations.json",
"events.json",
"scaffold.json",
]:
if (project_path / fixed).exists():
files.add(fixed)
# Keynote presentation(s)
for key_file in project_path.glob("*.key"):
files.add(key_file.name)
# NOTE: out/ (rendered mp4/srt) is deliberately NOT pushed. The rendering
# rig produces those; pushing the local (older) copy up would overwrite the
# rig's fresh render, which 'down' would then pull back — clobbering the new
# result. Rendered output flows one way only: rig → local via 'down'.
# Manuscript (may be at a non-standard path)
manuscript_rel = project.get("manuscript", "manuscript.txt")
if (project_path / manuscript_rel).exists():
files.add(manuscript_rel)
# Slide images + slides.json.
# The renderer reads this path lower-cased (parse_slides uses
# config.slides_path.lower()), because import may embed a capital-cased
# project name (e.g. "media/slides/Video3/" while the dir is "video3/").
# Push to the SAME lower-cased location — otherwise on a case-sensitive
# server 'up' writes media/slides/Video3/slides.json while render reads
# media/slides/video3/slides.json and never sees the update.
slides_rel = project.get("slides", "media/slides").lower()
slides_path = project_path / slides_rel
# slides might point directly to slides.json — include its parent dir
if slides_path.is_file():
files.add(slides_rel)
slides_path = slides_path.parent
if slides_path.is_dir():
for f in slides_path.rglob("*"):
if f.is_file():
files.add(str(f.relative_to(project_path)))
# Narration
# Push narration.json (preserves per-segment trim/channel settings) plus the
# RAW recordings in raw_mov/. Preprocessing runs on the rig — it produces the
# large processed/*_processed.mov segments there. We deliberately do NOT push
# each entry's source_file: after a local preprocess that points at
# processed/..._processed.mov, and uploading those would both waste bandwidth
# on files the rig regenerates and let a stale local render input clobber the
# rig's freshly produced one.
narration_rel = project.get("narration", "media/narration/narration.json")
narration_json = project_path / narration_rel
narration_dir = narration_json.parent
if narration_json.exists():
files.add(narration_rel)
raw_mov_dir = narration_dir / "raw_mov"
if raw_mov_dir.is_dir():
for f in sorted(raw_mov_dir.glob("*.mov")):
if f.is_file() and not f.name.startswith("."):
files.add(str(f.relative_to(project_path)))
# Per-segment Whisper transcripts. These are the ONLY non-deterministic input
# to marker alignment (Whisper is not bit-reproducible across platforms), so
# syncing them makes `build` produce identical events.json on any machine — the
# rig can no longer diverge by re-transcribing locally.
transcripts_dir = narration_dir / "transcripts"
if transcripts_dir.is_dir():
for f in sorted(transcripts_dir.glob("*.json")):
if f.is_file() and not f.name.startswith("."):
files.add(str(f.relative_to(project_path)))
# Audio — standard location, non-shared entries only
audio_json = project_path / "media" / "audio" / "audio.json"
if audio_json.exists():
files.add("media/audio/audio.json")
try:
data = json.loads(audio_json.read_text(encoding="utf-8"))
audio_dir = audio_json.parent
for entry in data.values():
if entry.get("is_shared"):
continue
src = entry.get("file")
if src:
candidate = audio_dir / src
if candidate.exists():
files.add(str(candidate.relative_to(project_path)))
except (json.JSONDecodeError, OSError):
pass
# Videos — non-shared entries only
videos_rel = project.get("videos", "media/videos/videos.json")
videos_json = project_path / videos_rel
videos_dir = videos_json.parent
if videos_json.exists():
files.add(videos_rel)
try:
data = json.loads(videos_json.read_text(encoding="utf-8"))
for entry in data.values():
if entry.get("is_shared"):
continue
src = entry.get("source_file")
if src:
candidate = videos_dir / src
if candidate.exists():
files.add(str(candidate.relative_to(project_path)))
except (json.JSONDecodeError, OSError):
pass
return sorted(files)
# ---------------------------------------------------------------------------
# Shared assets manifest
# Shared assets
# ---------------------------------------------------------------------------
def _find_shared_assets_root(project_path: Path) -> Optional[Path]:
@@ -208,69 +78,6 @@ def _find_shared_assets_root(project_path: Path) -> Optional[Path]:
return None
def _build_shared_manifest(project_path: Path) -> tuple:
"""Return (shared_assets_root, sorted list of root-relative paths) for shared files used by this project.
Layout conventions discovered from the actual filesystem:
- Shared videos: shared_assets/{source_file} (e.g. Logo6sec.mov, pexels/14923961.mp4)
- Shared audio: shared_assets/media/audio/{file}
"""
shared_root = _find_shared_assets_root(project_path)
if shared_root is None:
return None, []
files: set[str] = set()
# shared_assets/videos.json so the render pipeline can read metadata
if (shared_root / "videos.json").exists():
files.add("videos.json")
project: dict = {}
proj_json = project_path / "project.json"
if proj_json.exists():
try:
project = json.loads(proj_json.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
pass
# Shared video source files
videos_rel = project.get("videos", "media/videos/videos.json")
videos_json = project_path / videos_rel
if videos_json.exists():
try:
data = json.loads(videos_json.read_text(encoding="utf-8"))
for entry in data.values():
if not entry.get("is_shared"):
continue
src = entry.get("source_file")
if not src:
continue
# source_file is relative to shared_assets root
if (shared_root / src).exists():
files.add(src)
except (json.JSONDecodeError, OSError):
pass
# Shared audio source files (live under media/audio/ within shared_assets)
audio_json = project_path / "media" / "audio" / "audio.json"
if audio_json.exists():
try:
data = json.loads(audio_json.read_text(encoding="utf-8"))
for entry in data.values():
if not entry.get("is_shared"):
continue
src = entry.get("file")
if not src:
continue
rel = f"media/audio/{src}"
if (shared_root / rel).exists():
files.add(rel)
except (json.JSONDecodeError, OSError):
pass
return shared_root, sorted(files)
# ---------------------------------------------------------------------------
# Commit log helpers
# ---------------------------------------------------------------------------
@@ -335,59 +142,58 @@ def cmd_up(project_path: Path, verbose: bool, dry_run: bool) -> int:
print(f" Run: gnommo -p {project_path.name} down")
return 1
# Build manifest
manifest = _build_manifest(project_path)
total = len(manifest)
print(f"Pushing: {project_path.name} ({total} files, last commit: {local_last})")
if verbose:
for f in manifest:
print(f" {f}")
print(f"Pushing: {project_path.name} (whole tree minus excludes, last commit: {local_last})")
# Build shared assets manifest
shared_root, shared_files = _build_shared_manifest(project_path)
shared_root = _find_shared_assets_root(project_path)
remote_shared = f"{server['path']}/shared_assets"
if dry_run:
print("\n[DRY RUN] Would push project files:")
for f in manifest:
print(f" {f}")
if shared_files:
print(f"\n[DRY RUN] Would push shared assets ({len(shared_files)} files):")
for f in shared_files:
print(f" {f}")
return 0
# Pass 1: project files
subprocess.run([*ssh_cmd, f"mkdir -p {remote_project}"], check=True)
# Pass 1: project files — whole tree, denylist excludes.
rsync_cmd = [
"rsync", "-av", "--progress",
"--files-from=-",
"-e", f"ssh -p {server['port']}",
*[f"--exclude={p}" for p in _SYNC_EXCLUDES],
f"{project_path}/",
f"{server['user']}@{server['host']}:{remote_project}/",
]
result = subprocess.run(rsync_cmd, input="\n".join(manifest), text=True)
# Pass 2: shared assets — whole tree, same excludes.
rsync_shared = [
"rsync", "-av", "--progress",
"-e", f"ssh -p {server['port']}",
*[f"--exclude={p}" for p in _SYNC_EXCLUDES],
f"{shared_root}/" if shared_root else "",
f"{server['user']}@{server['host']}:{remote_shared}/",
]
if dry_run:
dry_project = rsync_cmd[:1] + ["--dry-run"] + rsync_cmd[1:]
print("\n[DRY RUN] Would push project:")
print(f" {' '.join(dry_project)}")
if shared_root:
dry_shared = rsync_shared[:1] + ["--dry-run"] + rsync_shared[1:]
print("\n[DRY RUN] Would push shared assets:")
print(f" {' '.join(dry_shared)}")
return 0
if verbose:
print(f" {' '.join(rsync_cmd)}")
# Pass 1: project files
subprocess.run([*ssh_cmd, f"mkdir -p {remote_project}"], check=True)
result = subprocess.run(rsync_cmd)
if result.returncode != 0:
print(f"Error: rsync failed for project files (code {result.returncode})")
return 1
# Pass 2: shared assets referenced by this project
if shared_files and shared_root:
print(f"\nPushing shared assets ({len(shared_files)} files)...")
# Pass 2: shared assets
if shared_root:
print("\nPushing shared assets...")
subprocess.run([*ssh_cmd, f"mkdir -p {remote_shared}"], check=True)
rsync_shared = [
"rsync", "-av", "--progress",
"--files-from=-",
"-e", f"ssh -p {server['port']}",
f"{shared_root}/",
f"{server['user']}@{server['host']}:{remote_shared}/",
]
result = subprocess.run(rsync_shared, input="\n".join(shared_files), text=True)
result = subprocess.run(rsync_shared)
if result.returncode != 0:
print(f"Error: rsync failed for shared assets (code {result.returncode})")
return 1
print(f"\nDone. Pushed {total} project files, {len(shared_files)} shared assets.")
print("\nDone.")
return 0
@@ -425,7 +231,7 @@ def cmd_down(project_path: Path, verbose: bool, dry_run: bool) -> int:
rsync_cmd = [
"rsync", "-av", "--progress",
"-e", f"ssh -p {server['port']}",
*[f"--exclude={p}" for p in _DOWN_EXCLUDES],
*[f"--exclude={p}" for p in _SYNC_EXCLUDES],
f"{server['user']}@{server['host']}:{remote_project}/",
f"{project_path}/",
]
@@ -435,7 +241,7 @@ def cmd_down(project_path: Path, verbose: bool, dry_run: bool) -> int:
rsync_shared_cmd = [
"rsync", "-av", "--progress",
"-e", f"ssh -p {server['port']}",
*[f"--exclude={p}" for p in _DOWN_EXCLUDES],
*[f"--exclude={p}" for p in _SYNC_EXCLUDES],
f"{server['user']}@{server['host']}:{remote_shared}/",
f"{local_shared}/",
]