gitprov/gnommo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding pexels downloader and fixes

Browse Source
This commit is contained in:
gitprov
2026-06-07 11:19:19 +02:00
parent 980bb84dac
commit b9b5a8e77d
12 changed files with 957 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gnommo/cli.py
+303 -54
View File
@@ -106,6 +106,7 @@ Examples:
"pull",
"handoff",
"transcode",
"pexels",
],
help="Action to perform (default: render)",
)
@@ -310,6 +311,8 @@ Examples:
return cmd_handoff(
project_path, args.verbose, args.file, args.prod, args.res
)
elif action == "pexels":
return cmd_pexels(project_path, args.verbose)
except GnommoError as e:
print(f"Error: {e}", file=sys.stderr)
@@ -362,7 +365,7 @@ def cmd_import(project_path: Path, force: bool, verbose: bool) -> int:
keynote_file = keynote_files[0] # Use first .key file found
if len(keynote_files) > 1:
print(f" Warning: Multiple .key files found, using {keynote_file.name}")
_import_presenter_notes(project_path, keynote_file, verbose)
_import_presenter_notes(project_path, keynote_file, verbose, config)
# Generate slides.json for each slide directory (after Keynote export)
slides_base = project_path / "media" / "slides"
@@ -391,6 +394,42 @@ def cmd_import(project_path: Path, force: bool, verbose: bool) -> int:
# Probe and cache video metadata (duration, has_audio) into videos.json
_probe_video_metadata(project_path, config, shared_assets_dir, force, verbose)
# ETL: if a manuscript exists, project shorthand marker semantics (cutout/layer)
# into videos.json so the render stage is always data-driven from the manuscript.
# Run AFTER sync so newly-added shared videos are already present when we write
# their cutout/layer. Also warn about any referenced video that is still missing.
manuscript_path = project_path / "manuscript.txt"
if manuscript_path.exists() and config:
from .parser import parse_manuscript
from .transformer import _SHORTHAND_PREFIXES
_, markers, _, _ = parse_manuscript(project_path)
if markers:
_project_markers_to_videos(
markers,
project_path / config.videos_path,
config,
project_path,
)
# Warn about shorthand-referenced videos still absent from videos.json
videos_json_path = project_path / config.videos_path
local_vids: dict = (
_read_json(videos_json_path) if videos_json_path.exists() else {}
)
seen_missing: set[str] = set()
for marker in markers:
for prefix in _SHORTHAND_PREFIXES:
if marker.startswith(prefix):
vid_id = marker[len(prefix):]
if vid_id not in local_vids and vid_id not in seen_missing:
print(
f" ⚠ [{marker}] video '{vid_id}' not found in "
f"videos.json or shared_assets — add it manually"
)
seen_missing.add(vid_id)
break
print("Import complete.")
return 0
@@ -729,33 +768,47 @@ def _import_shared_assets(shared_assets_dir: Path, verbose: bool) -> None:
"""
video_extensions = {".mov", ".mp4", ".webm", ".avi", ".mkv", ".m4v"}
# Find all video files in shared_assets (root level and subdirectories)
# Find all video files in shared_assets (root level and subdirectories).
# Also scan the GnommoDisk cache mirror so files placed there are registered.
from .cache import load_cache_config
scan_roots: list[Path] = [shared_assets_dir]
cache_base = load_cache_config()
if cache_base:
cache_shared = cache_base / "shared_assets"
if cache_shared.exists() and cache_shared != shared_assets_dir:
scan_roots.append(cache_shared)
video_files: list[tuple[Path, Path]] = [] # (relative_path, absolute_path)
seen_rel: set[str] = set() # deduplicate by relative path
for item in shared_assets_dir.iterdir():
if item.name.startswith("."):
continue
for scan_root in scan_roots:
for item in scan_root.iterdir():
if item.name.startswith("."):
continue
if item.is_file():
# Video file directly in shared_assets root
if (
item.suffix.lower() in video_extensions
and not item.name.endswith("_processed.mov")
and not item.name.endswith("_processed.webm")
):
rel_path = item.relative_to(shared_assets_dir)
video_files.append((rel_path, item))
elif item.is_dir():
# Scan subdirectories recursively
for video_file in item.rglob("*"):
if item.is_file():
if (
video_file.is_file()
and video_file.suffix.lower() in video_extensions
and not video_file.name.endswith("_processed.mov")
and not video_file.name.endswith("_processed.webm")
item.suffix.lower() in video_extensions
and not item.name.endswith("_processed.mov")
and not item.name.endswith("_processed.webm")
):
rel_path = video_file.relative_to(shared_assets_dir)
video_files.append((rel_path, video_file))
rel_path = item.relative_to(scan_root)
if str(rel_path) not in seen_rel:
seen_rel.add(str(rel_path))
video_files.append((rel_path, item))
elif item.is_dir():
for video_file in item.rglob("*"):
if (
video_file.is_file()
and video_file.suffix.lower() in video_extensions
and not video_file.name.endswith("_processed.mov")
and not video_file.name.endswith("_processed.webm")
):
rel_path = video_file.relative_to(scan_root)
if str(rel_path) not in seen_rel:
seen_rel.add(str(rel_path))
video_files.append((rel_path, video_file))
if not video_files:
if verbose:
@@ -1049,11 +1102,36 @@ def _import_narration_segments(narration_dir: Path, config, verbose: bool) -> No
print(f" No new narration segments to add")
def _write_youtube_meta(
project_path: Path, config, citations: list[str]
) -> None:
"""Write youtube_meta.txt with project description and collected citations."""
meta_path = project_path / "youtube_meta.txt"
lines: list[str] = []
if config and config.description:
lines.append("== Description ==")
lines.append(config.description)
lines.append("")
if citations:
lines.append("== References ==")
for i, cite in enumerate(citations, 1):
lines.append(f"{i}. {cite}")
lines.append("")
meta_path.write_text("\n".join(lines), encoding="utf-8")
print(f" Wrote {meta_path.name} ({len(citations)} reference(s))")
def _import_presenter_notes(
project_path: Path, keynote_file: Path, verbose: bool
project_path: Path, keynote_file: Path, verbose: bool, config=None
) -> None:
"""Extract presenter notes from Keynote and write to manuscript.txt.
[cite:...] markers are stripped from the manuscript and collected into
youtube_meta.txt alongside the project description.
Uses the JXA script (extract_keynote_notes.js) to extract notes via osascript.
Also exports slides as PNG images to media/slides/{project_name}/.
Backs up existing manuscript.txt before overwriting.
@@ -1116,21 +1194,44 @@ def _import_presenter_notes(
print(f" Error parsing notes JSON: {e}", file=sys.stderr)
return
# Convert to manuscript.txt format
# Convert to manuscript.txt format, stripping [cite:...] markers
_CITE_RE = re.compile(r"\[cite:([^\]]+)\]")
lines = []
citations: list[str] = []
seen_citations: set[str] = set()
for item in notes_data:
idx = item.get("slide_index")
notes = (item.get("notes") or "").rstrip()
lines.append(f"[S{idx}]")
if notes:
lines.append(notes)
clean_note_lines = []
for note_line in notes.splitlines():
for m in _CITE_RE.finditer(note_line):
cite_text = m.group(1).strip()
if cite_text not in seen_citations:
citations.append(cite_text)
seen_citations.add(cite_text)
cleaned = _CITE_RE.sub("", note_line).strip()
if cleaned:
clean_note_lines.append(cleaned)
if clean_note_lines:
lines.append("\n".join(clean_note_lines))
lines.append("") # blank line between slides
# Write manuscript.txt
manuscript_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
# Write manuscript.txt with Unix line endings (Keynote notes may contain \r\n or \r)
content = "\n".join(lines).rstrip() + "\n"
content = content.replace("\r\n", "\n").replace("\r", "\n")
manuscript_path.write_text(content, encoding="utf-8")
print(f" Wrote {manuscript_path} ({len(notes_data)} slides)")
# Write youtube_meta.txt with description + collected citations
_write_youtube_meta(project_path, config, citations)
if citations and verbose:
for i, cite in enumerate(citations, 1):
print(f" {i}. {cite}")
if verbose:
non_empty = sum(1 for item in notes_data if item.get("notes"))
print(f" {non_empty} slides have presenter notes")
@@ -1221,6 +1322,71 @@ def _write_tasks_file(
)
# =============================================================================
# Pexels Download Command
# =============================================================================
def cmd_pexels(project_path: Path, verbose: bool) -> int:
"""Download missing Pexels videos and enrich metadata for existing ones."""
from .parser import parse_manuscript, parse_project_config, parse_videos
from .pexels import (
get_pexels_api_key,
find_missing_pexels_videos,
download_video,
update_videos_json,
enrich_missing_descriptions,
)
api_key = get_pexels_api_key()
if not api_key:
print(
"Error: Pexels API key not configured.\n"
"Add to ~/.gnommo.conf:\n"
" [pexels]\n"
" api_key = YOUR_KEY_HERE\n"
"Get a free key at https://www.pexels.com/api/",
file=sys.stderr,
)
return 1
config = parse_project_config(project_path)
_, markers, _, _ = parse_manuscript(project_path)
videos, _ = parse_videos(project_path, config)
shared_assets_dir = _find_shared_assets(project_path)
if not shared_assets_dir:
print("Error: shared_assets directory not found.", file=sys.stderr)
return 1
local_videos_json = project_path / config.videos_path
shared_videos_json = shared_assets_dir / "videos.json"
# 1. Download missing files
missing = find_missing_pexels_videos(markers, videos, shared_assets_dir)
failed = 0
if missing:
print(f"Downloading {len(missing)} missing Pexels video(s)...")
for video_id, source_file in missing:
meta = download_video(source_file, shared_assets_dir, api_key)
if meta is None:
failed += 1
continue
for json_path in (local_videos_json, shared_videos_json):
update_videos_json(json_path, video_id, meta)
if failed:
print(f"\n {failed}/{len(missing)} download(s) failed.")
else:
print(f"\n {len(missing)} video(s) downloaded.")
else:
print("No missing Pexels videos.")
# 2. Enrich descriptions for existing files that have none
enrich_missing_descriptions(shared_assets_dir, api_key)
return 1 if failed else 0
# =============================================================================
# Validate Command
# =============================================================================
@@ -1283,6 +1449,35 @@ def _resolve_process_cache(project_path: Path, config) -> Optional[Path]:
return p / project_path.name
def _narration_combined_hint(project_path: Path, config) -> str:
"""Return a helpful hint when narration_combined.mov cannot be found.
If external storage is configured but the volume isn't mounted, the stitch
command wouldn't help — the disk is just not connected.
"""
from .cache import load_cache_config
missing_paths = []
cache_base = load_cache_config()
if cache_base is not None and not cache_base.exists():
missing_paths.append(cache_base)
if config and config.process_cache:
pc = Path(config.process_cache)
if not pc.is_absolute():
pc = (project_path / pc).resolve()
if not pc.exists():
missing_paths.append(pc)
if missing_paths:
return (
f"External disk not connected (expected at {missing_paths[0]}).\n"
"Connect the disk and try again."
)
return "Run 'gnommo -p <project> stitch' first."
def _resolve_narration_combined(
project_path: Path, videos_dir: Path, config
) -> Optional[Path]:
@@ -2143,8 +2338,8 @@ def cmd_stitch(
videos_dir_out.mkdir(parents=True, exist_ok=True)
print(f" Using {res} dirs: {narration_dir}, {videos_dir_out}")
# Get segment IDs in sorted order
segment_ids = sorted(narration.keys())
# Get segment IDs in natural order (Segment2 before Segment10)
segment_ids = sorted(narration.keys(), key=lambda s: [int(t) if t.isdigit() else t.lower() for t in re.split(r'(\d+)', s)])
# Show what we're stitching
print(f"\n Segments ({len(segment_ids)}):")
@@ -2442,7 +2637,7 @@ def _parse_slide_range(slides_arg: str) -> tuple[str, Optional[str]]:
def _project_markers_to_videos(
markers: list[str], videos_json_path: Path, config
markers: list[str], videos_json_path: Path, config, project_path: Path = None
) -> None:
"""ETL: project shorthand marker semantics into videos.json.
@@ -2451,6 +2646,9 @@ def _project_markers_to_videos(
and layer values directly into videos.json. This runs before parse_videos
so the render pass reads already-projected data and needs no shorthand logic.
Videos may live in the project's local videos.json or in shared_assets/videos.json.
Both files are updated so the render pass always finds the projected values.
The manuscript is the authoritative source: the LAST shorthand reference to
a given video_id wins, matching what a human editor would expect when they
change a marker near the end of the script.
@@ -2460,36 +2658,75 @@ def _project_markers_to_videos(
from .transformer import _SHORTHAND_PREFIXES # (cutout, layer) lookup table
# Build projection: video_id → {cutout, layer}
_PAUSE_PREFIXES = {
"vftp:", "vfbp:", "vfmp:",
"vf2tp:", "vf2bp:", "vf2mp:",
"vstp:", "vsbp:", "vsmp:",
}
# Build projection: video_id → {cutout, layer, auto_pause_narration}
# auto_pause_narration=True means: write pause_narration=duration if not already set.
projection: dict[str, dict] = {}
for marker in markers:
for prefix, implied in _SHORTHAND_PREFIXES.items():
if marker.startswith(prefix):
video_id = marker[len(prefix) :]
video_id = marker[len(prefix):]
cutout, layer = implied[0], implied[1]
projection[video_id] = {"cutout": cutout, "layer": layer}
projection[video_id] = {
"cutout": cutout,
"layer": layer,
"_auto_pause": prefix in _PAUSE_PREFIXES,
}
break
if not projection:
return
with open(videos_json_path, "r", encoding="utf-8") as f:
raw = json.load(f)
def _apply_projection(json_path: Path) -> list[str]:
"""Apply projection to one videos.json file; return list of updated IDs."""
if not json_path.exists():
return []
with open(json_path, "r", encoding="utf-8") as f:
raw = json.load(f)
changed = False
updated = []
for video_id, fields in projection.items():
if video_id not in raw:
continue
entry = raw[video_id]
video_changed = False
for field, value in fields.items():
if field == "_auto_pause":
# Write pause_narration = duration only when:
# - marker is a pause-prefix (value is True)
# - pause_narration not already set (preserve manual overrides)
# - duration is known (probed by import)
if value and not entry.get("pause_narration") and entry.get("duration"):
entry["pause_narration"] = entry["duration"]
changed = True
video_changed = True
elif entry.get(field) != value:
entry[field] = value
changed = True
video_changed = True
if video_changed:
updated.append(video_id)
if changed:
with open(json_path, "w", encoding="utf-8") as f:
json.dump(raw, f, indent=2, ensure_ascii=False)
return updated
changed = False
for video_id, fields in projection.items():
if video_id not in raw:
continue
for field, value in fields.items():
if raw[video_id].get(field) != value:
raw[video_id][field] = value
changed = True
updated_local = _apply_projection(videos_json_path)
if updated_local:
print(f" Projected marker semantics → videos.json: {', '.join(updated_local)}")
if changed:
with open(videos_json_path, "w", encoding="utf-8") as f:
json.dump(raw, f, indent=2, ensure_ascii=False)
updated = [vid for vid in projection if vid in raw]
print(f" Projected marker semantics → videos.json: {', '.join(updated)}")
# Also project into shared_assets/videos.json for pexels/library videos
shared_assets_dir = _find_shared_assets(project_path) if project_path else None
if shared_assets_dir:
shared_videos_json = shared_assets_dir / "videos.json"
updated_shared = _apply_projection(shared_videos_json)
if updated_shared:
print(f" Projected marker semantics → shared_assets/videos.json: {', '.join(updated_shared)}")
def _writeback_video_metadata(plan, project_path, config) -> None:
@@ -2696,7 +2933,7 @@ def cmd_render(
# ETL: project shorthand marker semantics (cutout/layer) into videos.json
# before parse_videos reads it, so the render pass is purely data-driven.
_project_markers_to_videos(markers, project_path / config.videos_path, config)
_project_markers_to_videos(markers, project_path / config.videos_path, config, project_path)
# Override resolution for preview modes
if res != "full":
@@ -2705,6 +2942,7 @@ def cmd_render(
slides = parse_slides(project_path, config)
videos, videos_dir = parse_videos(project_path, config)
source_videos_dir = videos_dir # keep original for validation (pre-downscale)
# Non-full res: use downscaled video directory, create on-the-fly if needed
if res != "full":
@@ -2807,6 +3045,12 @@ def cmd_render(
else:
transcript_path = project_path / "transcript.json"
# If project.json specifies a transcript path, prefer it (always local)
if config.transcript_path:
local_transcript = project_path / config.transcript_path
if local_transcript.exists():
transcript_path = local_transcript
# Try cache fallback for transcript
transcript_path, _ = resolve_with_cache(transcript_path, project_path)
if not transcript_path.exists():
@@ -2825,7 +3069,7 @@ def cmd_render(
# Stage 2: Validate
print("\n[2/4] Validating...")
warnings = validate_project(
project_path, markers, config, slides, videos, videos_dir, malformed
project_path, markers, config, slides, videos, source_videos_dir, malformed
)
for w in warnings:
print(f" Warning: {w}")
@@ -3061,7 +3305,12 @@ def cmd_transcribe(
words = transcribe_video(video_path, model="base")
output_path = video_path.with_suffix(".transcript.json")
# Save to project-local path if configured in project.json (keeps transcript off external drives)
if config.transcript_path:
output_path = project_path / config.transcript_path
output_path.parent.mkdir(parents=True, exist_ok=True)
else:
output_path = video_path.with_suffix(".transcript.json")
save_transcript(words, output_path)
print(f" - Transcribed {len(words)} words")
@@ -3819,7 +4068,7 @@ def cmd_extract_audio(
f"Error: narration_combined.mov not found at {combined_path}",
file=sys.stderr,
)
print("Run 'gnommo -p <project> stitch' first.", file=sys.stderr)
print(_narration_combined_hint(project_path, config), file=sys.stderr)
return 1
# Output to project out/ directory
@@ -3985,7 +4234,7 @@ def cmd_master(
f"Error: narration_combined.mov not found at {combined_path}",
file=sys.stderr,
)
print("Run 'gnommo -p <project> stitch' first.", file=sys.stderr)
print(_narration_combined_hint(project_path, config), file=sys.stderr)
return 1
# Output directory