Add transcription and alignment pipeline

New commands:
- `transcribe`: Uses Whisper to generate word-level timestamps from video
- `align`: Matches manuscript markers to transcript, outputs transcript.csv

Workflow:
1. gnommo transcribe video.mov → video.transcript.json
2. gnommo align project/ → transcript.csv with markers at aligned times

Alignment uses fuzzy text matching to find the first phrase after each
marker in the manuscript, then locates it in the transcript. Applies
configurable offset (default -1s) so slides appear before speech.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-01-12 12:41:10 +01:00
co-authored by Claude Opus 4.5
parent 7f7425da46
commit 216131e072
4 changed files with 418 additions and 0 deletions
+127
View File
@@ -18,6 +18,8 @@ from .parser import (
from .validator import validate_project
from .transformer import build_render_plan
from .renderer import render, generate_ffmpeg_command_string
from .transcriber import transcribe_video, save_transcript, load_transcript
from .aligner import align_markers, save_aligned_transcript
def main() -> int:
@@ -87,6 +89,50 @@ def main() -> int:
help="Slide type for all slides (default: square)",
)
# transcribe command
transcribe_parser = subparsers.add_parser(
"transcribe",
help="Transcribe video audio using Whisper",
)
transcribe_parser.add_argument(
"video",
type=Path,
help="Path to video file",
)
transcribe_parser.add_argument(
"-o", "--output",
type=Path,
help="Output JSON file (default: <video>.transcript.json)",
)
transcribe_parser.add_argument(
"--model",
default="base",
choices=["tiny", "base", "small", "medium", "large"],
help="Whisper model size (default: base)",
)
# align command
align_parser = subparsers.add_parser(
"align",
help="Align manuscript markers to transcript timestamps",
)
align_parser.add_argument(
"project",
type=Path,
help="Path to project directory",
)
align_parser.add_argument(
"--transcript",
type=Path,
help="Path to transcript JSON (default: media/talking_head.transcript.json)",
)
align_parser.add_argument(
"--offset",
type=float,
default=-1.0,
help="Seconds to offset marker times (default: -1.0)",
)
args = parser.parse_args()
try:
@@ -97,6 +143,11 @@ def main() -> int:
return cmd_render(args.project, output, args.verbose, args.dry_run)
elif args.command == "generate-slides":
return cmd_generate_slides(args.directory, args.type)
elif args.command == "transcribe":
output = args.output or args.video.with_suffix(".transcript.json")
return cmd_transcribe(args.video, output, args.model)
elif args.command == "align":
return cmd_align(args.project, args.transcript, args.offset)
except GnommoError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
@@ -231,5 +282,81 @@ def cmd_generate_slides(directory: Path, slide_type: str) -> int:
return 0
def cmd_transcribe(video_path: Path, output_path: Path, model: str) -> int:
"""Transcribe video audio using Whisper."""
print(f"Transcribing: {video_path}")
print(f"Model: {model}")
print()
words = transcribe_video(video_path, model=model)
print(f" - Transcribed {len(words)} words")
print(f" - Duration: {words[-1].end:.1f}s" if words else " - No words found")
save_transcript(words, output_path)
print(f" - Saved to: {output_path}")
# Show first few words as preview
if words:
preview = " ".join(w.word for w in words[:10])
print(f" - Preview: {preview}...")
return 0
def cmd_align(project_path: Path, transcript_path: Path = None, offset: float = -1.0) -> int:
"""Align manuscript markers to transcript timestamps."""
print(f"Aligning: {project_path}")
print(f"Offset: {offset}s")
print()
# Load manuscript
manuscript_path = project_path / "manuscript.txt"
if not manuscript_path.exists():
print(f"Error: manuscript.txt not found", file=sys.stderr)
return 1
manuscript_text = manuscript_path.read_text(encoding="utf-8")
# Load transcript
if transcript_path is None:
# Try to find transcript in media folder
transcript_path = project_path / "media" / "talking_head.transcript.json"
if not transcript_path.exists():
print(f"Error: Transcript not found: {transcript_path}", file=sys.stderr)
print("Run 'gnommo transcribe' first to generate the transcript.", file=sys.stderr)
return 1
print(f" - Loading transcript: {transcript_path}")
transcript = load_transcript(transcript_path)
print(f" - Loaded {len(transcript)} words")
# Align markers
print(" - Aligning markers...")
alignments = align_markers(manuscript_text, transcript, offset_seconds=offset)
# Report results
print()
print("Alignment results:")
unmatched = 0
for a in alignments:
if a.timestamp >= 0:
print(f" [{a.marker_id}] @ {a.timestamp:.2f}s - \"{a.matched_phrase}...\"")
else:
print(f" [{a.marker_id}] NOT FOUND - \"{a.matched_phrase}...\"")
unmatched += 1
if unmatched > 0:
print(f"\nWarning: {unmatched} markers could not be aligned")
# Save aligned transcript.csv
output_path = project_path / "transcript.csv"
save_aligned_transcript(alignments, transcript, output_path)
print(f"\nSaved: {output_path}")
return 0
if __name__ == "__main__":
sys.exit(main())