diff --git a/gnommo/cli.py b/gnommo/cli.py index 7521151..c7dc144 100644 --- a/gnommo/cli.py +++ b/gnommo/cli.py @@ -57,10 +57,12 @@ Examples: gnommo -p video1 description Generate YouTube description file gnommo -p video1 transcribe Narration file for timing of slides gnommo -p video1 transcribe --final Transcribe outputted file and generate SRT for YouTube - gnommo -p video1 archive Sync project to external cache storage - gnommo -p video1 archive --dry-run Preview what would be synced - gnommo -p video1 up Upload project files to remote server - gnommo -p video1 down Download project files from remote server + gnommo -p video1 archive Copy project to connected external drive + gnommo -p video1 load Copy project from external drive to local + gnommo -p video1 commit -m "msg" Record a commit to commits.log (required before up) + gnommo -p video1 up Push manifest files to rendering server + gnommo -p video1 up --dry-run Preview which files would be pushed + gnommo -p video1 down Pull files from rendering server to local gnommo -p video1 extract-audio --combined Extract audio from narration_combined.mov gnommo -p video1 extract-audio --combined --channel left Extract left channel only gnommo -p video1 extract-audio --segment seg01 Extract from a specific segment @@ -100,6 +102,7 @@ Examples: "description", "archive", "load", + "commit", "up", "down", "extract-audio", @@ -226,6 +229,13 @@ Examples: dest="alpha_quality", help="For transcode --processed: HEVC alpha quality 0.0-1.0 (default: 0.75; lower=smaller file)", ) + parser.add_argument( + "-m", + "--message", + type=str, + default=None, + help="For commit: commit message", + ) parser.add_argument( "--search", type=str, @@ -322,10 +332,18 @@ Examples: return cmd_archive(project_path, args.verbose, args.dry_run) elif action == "load": return cmd_load(project_path, args.verbose, args.dry_run) + elif action == "commit": + from .transfer import cmd_commit + if not args.message: + print("Error: -m 'message' is required for commit.") + return 1 + return cmd_commit(project_path, args.message) elif action == "up": - return cmd_sync(project_path, args.verbose, args.dry_run, download=False) + from .transfer import cmd_up + return cmd_up(project_path, args.verbose, args.dry_run) elif action == "down": - return cmd_sync(project_path, args.verbose, args.dry_run, download=True) + from .transfer import cmd_down + return cmd_down(project_path, args.verbose, args.dry_run) elif action == "extract-audio": return cmd_extract_audio( project_path, args.verbose, args.segment, args.channel, args.combined @@ -4407,7 +4425,8 @@ def cmd_all( return result print("\n>>> Step 8/8: Upload\n") - return cmd_sync(project_path, verbose, dry_run, download=False) + from .transfer import cmd_up + return cmd_up(project_path, verbose, dry_run) # ============================================================================= @@ -4710,84 +4729,6 @@ def cmd_load(project_path: Path, verbose: bool, dry_run: bool) -> int: return 0 -def cmd_sync(project_path: Path, verbose: bool, dry_run: bool, download: bool) -> int: - """Sync project files to/from the remote server via rsync over SSH.""" - from .cache import load_server_config - - server = load_server_config() - if server is None: - print("Error: Server not configured. Add to ~/.gnommo.conf:") - print(" [server]") - print(" host = 76.13.144.52") - print(" user = root") - print(" path = /gnommo/project") - return 1 - - direction = "Downloading from" if download else "Uploading to" - print(f"{direction} server: {project_path.name}") - - remote = f"{server['user']}@{server['host']}:{server['path']}/{project_path.name}/" - local = f"{project_path}/" - - if download: - src, dest = remote, local - else: - src, dest = local, remote - - print(f" Source: {src}") - print(f" Destination: {dest}") - - # Ensure destination directory exists - if not dry_run: - if download: - project_path.mkdir(parents=True, exist_ok=True) - else: - remote_dir = f"{server['path']}/{project_path.name}" - ssh_cmd = [ - "ssh", - "-p", - server["port"], - f"{server['user']}@{server['host']}", - f"mkdir -p {remote_dir}", - ] - if verbose: - print(f" Creating remote dir: {remote_dir}") - result = subprocess.run(ssh_cmd) - if result.returncode != 0: - print(f"Error: could not create remote directory {remote_dir}") - return 1 - - rsync_cmd = [ - "rsync", - "-av", - "--progress", - "-e", - f"ssh -p {server['port']}", - *[f"--exclude={p}" for p in _RSYNC_EXCLUDES], - # On upload: delete server-side files that no longer exist locally so - # the remote stays an exact mirror of the local project. - *(["--delete"] if not download else []), - src, - dest, - ] - - if dry_run: - rsync_cmd.insert(1, "--dry-run") - print("\n [DRY RUN] Would execute:") - print(f" {' '.join(rsync_cmd)}") - else: - print("\n Syncing files...") - - if verbose: - print(f" Command: {' '.join(rsync_cmd)}") - - result = subprocess.run(rsync_cmd) - if result.returncode != 0: - print(f"Error: rsync failed with code {result.returncode}") - return 1 - - print("\nDone.") - return 0 # ============================================================================= diff --git a/gnommo/transfer.py b/gnommo/transfer.py new file mode 100644 index 0000000..04c35ab --- /dev/null +++ b/gnommo/transfer.py @@ -0,0 +1,294 @@ +"""SSH-based file transfer for gnommo projects (commit / up / down). + +Workflow: + gnommo -p video1 commit -m "re-recorded slides 7-15" + gnommo -p video1 up # push to rendering server + gnommo -p video1 down # pull from rendering server + +Design: + - commit appends a timestamped entry to commits.log + - up checks server commits.log for newer entry (aborts if found), + then rsyncs a manifest of only the files needed to render + - down rsyncs everything the server has back to local + (server is already clean — it only holds what was pushed) +""" + +import json +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Optional + +_COMMITS_LOG = "commits.log" + +# Dirs/patterns excluded on down (mirrors what up never pushes). +_DOWN_EXCLUDES = [ + "media/narration/processed/", + "media/narration/intermediate/", + "media/videos/intermediate/", + "media/narration/low/", + "media/videos/low/", + "media/videos/narration_combined.mov", + "**/chunks/", + "*.tmp", + ".DS_Store", +] + + +# --------------------------------------------------------------------------- +# Manifest builder +# --------------------------------------------------------------------------- + +def _build_manifest(project_path: Path) -> list[str]: + """Return sorted list of project-relative paths required to render.""" + files: set[str] = set() + + # Read project.json for path overrides + proj_json = project_path / "project.json" + project: dict = {} + if proj_json.exists(): + try: + project = json.loads(proj_json.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + pass + + # Fixed project-root files + for fixed in ["project.json", "manuscript.txt", _COMMITS_LOG, "citations.json"]: + if (project_path / fixed).exists(): + files.add(fixed) + + # Manuscript (may be at a non-standard path) + manuscript_rel = project.get("manuscript", "manuscript.txt") + if (project_path / manuscript_rel).exists(): + files.add(manuscript_rel) + + # Slide images + slides.json + slides_rel = project.get("slides", "media/slides") + slides_path = project_path / slides_rel + # slides might point directly to slides.json — include its parent dir + if slides_path.is_file(): + files.add(slides_rel) + slides_path = slides_path.parent + if slides_path.is_dir(): + for f in slides_path.rglob("*"): + if f.is_file(): + files.add(str(f.relative_to(project_path))) + + # Narration + narration_rel = project.get("narration", "media/narration/narration.json") + narration_json = project_path / narration_rel + narration_dir = narration_json.parent + if narration_json.exists(): + files.add(narration_rel) + try: + data = json.loads(narration_json.read_text(encoding="utf-8")) + for entry in data.values(): + src = entry.get("source_file") + if src: + candidate = narration_dir / src + if candidate.exists(): + files.add(str(candidate.relative_to(project_path))) + except (json.JSONDecodeError, OSError): + pass + + # Audio — standard location, non-shared entries only + audio_json = project_path / "media" / "audio" / "audio.json" + if audio_json.exists(): + files.add("media/audio/audio.json") + try: + data = json.loads(audio_json.read_text(encoding="utf-8")) + audio_dir = audio_json.parent + for entry in data.values(): + if entry.get("is_shared"): + continue + src = entry.get("file") + if src: + candidate = audio_dir / src + if candidate.exists(): + files.add(str(candidate.relative_to(project_path))) + except (json.JSONDecodeError, OSError): + pass + + # Videos — non-shared entries only + videos_rel = project.get("videos", "media/videos/videos.json") + videos_json = project_path / videos_rel + videos_dir = videos_json.parent + if videos_json.exists(): + files.add(videos_rel) + try: + data = json.loads(videos_json.read_text(encoding="utf-8")) + for entry in data.values(): + if entry.get("is_shared"): + continue + src = entry.get("source_file") + if src: + candidate = videos_dir / src + if candidate.exists(): + files.add(str(candidate.relative_to(project_path))) + except (json.JSONDecodeError, OSError): + pass + + return sorted(files) + + +# --------------------------------------------------------------------------- +# Commit log helpers +# --------------------------------------------------------------------------- + +def _read_log_lines(path: Path) -> list[str]: + if not path.exists(): + return [] + return [l for l in path.read_text(encoding="utf-8").splitlines() if l.strip()] + + +def _last_timestamp(lines: list[str]) -> Optional[str]: + for line in reversed(lines): + ts = line.split(" | ")[0].strip() + if ts: + return ts + return None + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + +def cmd_commit(project_path: Path, message: str) -> int: + log_path = project_path / _COMMITS_LOG + ts = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + with open(log_path, "a", encoding="utf-8") as f: + f.write(f"{ts} | {message}\n") + print(f"[{ts}] {message}") + return 0 + + +def cmd_up(project_path: Path, verbose: bool, dry_run: bool) -> int: + from .cache import load_server_config + + server = load_server_config() + if server is None: + _print_server_error() + return 1 + + # Require a local commit + log_path = project_path / _COMMITS_LOG + local_lines = _read_log_lines(log_path) + if not local_lines: + print(f"Error: No commits. Run: gnommo -p {project_path.name} commit -m 'message'") + return 1 + local_last = _last_timestamp(local_lines) + + # Fetch server's commits.log and compare + remote_project = f"{server['path']}/{project_path.name}" + ssh_cmd = ["ssh", "-p", server["port"], f"{server['user']}@{server['host']}"] + result = subprocess.run( + [*ssh_cmd, f"cat {remote_project}/{_COMMITS_LOG} 2>/dev/null"], + capture_output=True, text=True, + ) + server_lines = [l for l in result.stdout.splitlines() if l.strip()] + server_last = _last_timestamp(server_lines) + + if server_last and server_last > local_last: + print(f"Error: Server has a more recent commit — pull first.") + print(f" Local: {local_last}") + print(f" Server: {server_last}") + print(f" Run: gnommo -p {project_path.name} down") + return 1 + + # Build manifest + manifest = _build_manifest(project_path) + total = len(manifest) + print(f"Pushing: {project_path.name} ({total} files, last commit: {local_last})") + if verbose: + for f in manifest: + print(f" {f}") + + if dry_run: + print("\n[DRY RUN] Would push:") + for f in manifest: + print(f" {f}") + return 0 + + # Ensure remote dir exists + subprocess.run([*ssh_cmd, f"mkdir -p {remote_project}"], check=True) + + rsync_cmd = [ + "rsync", "-av", "--progress", + "--files-from=-", + "-e", f"ssh -p {server['port']}", + f"{project_path}/", + f"{server['user']}@{server['host']}:{remote_project}/", + ] + result = subprocess.run(rsync_cmd, input="\n".join(manifest), text=True) + if result.returncode != 0: + print(f"Error: rsync failed (code {result.returncode})") + return 1 + + print(f"\nDone. Pushed {total} files.") + return 0 + + +def cmd_down(project_path: Path, verbose: bool, dry_run: bool) -> int: + from .cache import load_server_config + + server = load_server_config() + if server is None: + _print_server_error() + return 1 + + remote_project = f"{server['path']}/{project_path.name}" + + # Verify the project exists on the server + ssh_cmd = ["ssh", "-p", server["port"], f"{server['user']}@{server['host']}"] + result = subprocess.run( + [*ssh_cmd, f"test -d {remote_project} && echo ok"], + capture_output=True, text=True, + ) + if result.stdout.strip() != "ok": + print(f"Error: Project not found on server: {remote_project}") + return 1 + + remote_log_result = subprocess.run( + [*ssh_cmd, f"cat {remote_project}/{_COMMITS_LOG} 2>/dev/null"], + capture_output=True, text=True, + ) + server_lines = [l for l in remote_log_result.stdout.splitlines() if l.strip()] + server_last = _last_timestamp(server_lines) + + print(f"Pulling: {project_path.name} (server last commit: {server_last or 'none'})") + + project_path.mkdir(parents=True, exist_ok=True) + + rsync_cmd = [ + "rsync", "-av", "--progress", + "-e", f"ssh -p {server['port']}", + *[f"--exclude={p}" for p in _DOWN_EXCLUDES], + f"{server['user']}@{server['host']}:{remote_project}/", + f"{project_path}/", + ] + + if dry_run: + rsync_cmd.insert(1, "--dry-run") + print("\n[DRY RUN] Would execute:") + print(f" {' '.join(rsync_cmd)}") + return 0 + + if verbose: + print(f" {' '.join(rsync_cmd)}") + + result = subprocess.run(rsync_cmd) + if result.returncode != 0: + print(f"Error: rsync failed (code {result.returncode})") + return 1 + + print("\nDone.") + return 0 + + +def _print_server_error(): + print("Error: Server not configured. Add to ~/.gnommo.conf:") + print(" [server]") + print(" host = 192.168.1.100") + print(" user = username") + print(" path = /gnommo/project") + print(" port = 22")