gnommo/gnommo/transfer.py

"""SSH-based file transfer for gnommo projects (commit / up / down).

Workflow:
  gnommo -p video1 commit -m "re-recorded slides 7-15"
  gnommo -p video1 up        # push to rendering server
  gnommo -p video1 down      # pull from rendering server

Design:
  - commit  appends a timestamped entry to commits.log
  - up      checks server commits.log for newer entry (aborts if found),
            then rsyncs a manifest of only the *inputs* needed to render
            (manuscript, slides, narration, audio, videos, keynote). It does
            NOT push the rendered output (out/*.mp4/.srt) — that is produced
            on the rig, so pushing a stale local copy would clobber it.
  - down    rsyncs everything the server has back to local, including the
            freshly rendered out/*.mp4. Rendered output flows one way: rig → local.
"""

import json
import subprocess
from datetime import datetime
from pathlib import Path
from typing import Optional

_COMMITS_LOG = "commits.log"

# Dirs/patterns excluded on down (mirrors what up never pushes).
_DOWN_EXCLUDES = [
    "media/narration/processed/",
    "media/narration/intermediate/",
    "media/videos/intermediate/",
    "media/narration/low/",
    "media/videos/low/",
    "**/chunks/",
    "*.tmp",
    ".*",       # rsync in-progress temp files (.filename.XXXXXX) and .DS_Store
]


# ---------------------------------------------------------------------------
# Manifest builder
# ---------------------------------------------------------------------------

def _build_manifest(project_path: Path) -> list[str]:
    """Return sorted list of project-relative paths required to render."""
    files: set[str] = set()

    # Read project.json for path overrides
    proj_json = project_path / "project.json"
    project: dict = {}
    if proj_json.exists():
        try:
            project = json.loads(proj_json.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            pass

    # Fixed project-root files
    for fixed in ["project.json", "manuscript.txt", _COMMITS_LOG, "citations.json"]:
        if (project_path / fixed).exists():
            files.add(fixed)

    # Keynote presentation(s)
    for key_file in project_path.glob("*.key"):
        files.add(key_file.name)

    # NOTE: out/ (rendered mp4/srt) is deliberately NOT pushed. The rendering
    # rig produces those; pushing the local (older) copy up would overwrite the
    # rig's fresh render, which 'down' would then pull back — clobbering the new
    # result. Rendered output flows one way only: rig → local via 'down'.

    # Manuscript (may be at a non-standard path)
    manuscript_rel = project.get("manuscript", "manuscript.txt")
    if (project_path / manuscript_rel).exists():
        files.add(manuscript_rel)

    # Slide images + slides.json.
    # The renderer reads this path lower-cased (parse_slides uses
    # config.slides_path.lower()), because import may embed a capital-cased
    # project name (e.g. "media/slides/Video3/" while the dir is "video3/").
    # Push to the SAME lower-cased location — otherwise on a case-sensitive
    # server 'up' writes media/slides/Video3/slides.json while render reads
    # media/slides/video3/slides.json and never sees the update.
    slides_rel = project.get("slides", "media/slides").lower()
    slides_path = project_path / slides_rel
    # slides might point directly to slides.json — include its parent dir
    if slides_path.is_file():
        files.add(slides_rel)
        slides_path = slides_path.parent
    if slides_path.is_dir():
        for f in slides_path.rglob("*"):
            if f.is_file():
                files.add(str(f.relative_to(project_path)))

    # Narration
    narration_rel = project.get("narration", "media/narration/narration.json")
    narration_json = project_path / narration_rel
    narration_dir = narration_json.parent
    if narration_json.exists():
        files.add(narration_rel)
        try:
            data = json.loads(narration_json.read_text(encoding="utf-8"))
            for entry in data.values():
                src = entry.get("source_file")
                if src:
                    candidate = narration_dir / src
                    if candidate.exists():
                        files.add(str(candidate.relative_to(project_path)))
        except (json.JSONDecodeError, OSError):
            pass

    # Audio — standard location, non-shared entries only
    audio_json = project_path / "media" / "audio" / "audio.json"
    if audio_json.exists():
        files.add("media/audio/audio.json")
        try:
            data = json.loads(audio_json.read_text(encoding="utf-8"))
            audio_dir = audio_json.parent
            for entry in data.values():
                if entry.get("is_shared"):
                    continue
                src = entry.get("file")
                if src:
                    candidate = audio_dir / src
                    if candidate.exists():
                        files.add(str(candidate.relative_to(project_path)))
        except (json.JSONDecodeError, OSError):
            pass

    # Videos — non-shared entries only
    videos_rel = project.get("videos", "media/videos/videos.json")
    videos_json = project_path / videos_rel
    videos_dir = videos_json.parent
    if videos_json.exists():
        files.add(videos_rel)
        try:
            data = json.loads(videos_json.read_text(encoding="utf-8"))
            for entry in data.values():
                if entry.get("is_shared"):
                    continue
                src = entry.get("source_file")
                if src:
                    candidate = videos_dir / src
                    if candidate.exists():
                        files.add(str(candidate.relative_to(project_path)))
        except (json.JSONDecodeError, OSError):
            pass

    return sorted(files)


# ---------------------------------------------------------------------------
# Shared assets manifest
# ---------------------------------------------------------------------------

def _find_shared_assets_root(project_path: Path) -> Optional[Path]:
    """Return the local shared_assets directory, or None if not found."""
    candidate = project_path.parent / "shared_assets"
    if candidate.is_dir():
        return candidate
    try:
        from .cache import load_assets_config
        assets = load_assets_config()
        if assets:
            candidate = assets / "shared_assets"
            if candidate.is_dir():
                return candidate
    except Exception:
        pass
    return None


def _build_shared_manifest(project_path: Path) -> tuple:
    """Return (shared_assets_root, sorted list of root-relative paths) for shared files used by this project.

    Layout conventions discovered from the actual filesystem:
      - Shared videos:  shared_assets/{source_file}   (e.g. Logo6sec.mov, pexels/14923961.mp4)
      - Shared audio:   shared_assets/media/audio/{file}
    """
    shared_root = _find_shared_assets_root(project_path)
    if shared_root is None:
        return None, []

    files: set[str] = set()

    # shared_assets/videos.json so the render pipeline can read metadata
    if (shared_root / "videos.json").exists():
        files.add("videos.json")

    project: dict = {}
    proj_json = project_path / "project.json"
    if proj_json.exists():
        try:
            project = json.loads(proj_json.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            pass

    # Shared video source files
    videos_rel = project.get("videos", "media/videos/videos.json")
    videos_json = project_path / videos_rel
    if videos_json.exists():
        try:
            data = json.loads(videos_json.read_text(encoding="utf-8"))
            for entry in data.values():
                if not entry.get("is_shared"):
                    continue
                src = entry.get("source_file")
                if not src:
                    continue
                # source_file is relative to shared_assets root
                if (shared_root / src).exists():
                    files.add(src)
        except (json.JSONDecodeError, OSError):
            pass

    # Shared audio source files (live under media/audio/ within shared_assets)
    audio_json = project_path / "media" / "audio" / "audio.json"
    if audio_json.exists():
        try:
            data = json.loads(audio_json.read_text(encoding="utf-8"))
            for entry in data.values():
                if not entry.get("is_shared"):
                    continue
                src = entry.get("file")
                if not src:
                    continue
                rel = f"media/audio/{src}"
                if (shared_root / rel).exists():
                    files.add(rel)
        except (json.JSONDecodeError, OSError):
            pass

    return shared_root, sorted(files)


# ---------------------------------------------------------------------------
# Commit log helpers
# ---------------------------------------------------------------------------

def _read_log_lines(path: Path) -> list[str]:
    if not path.exists():
        return []
    return [l for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]


def _last_timestamp(lines: list[str]) -> Optional[str]:
    for line in reversed(lines):
        ts = line.split(" | ")[0].strip()
        if ts:
            return ts
    return None


# ---------------------------------------------------------------------------
# Commands
# ---------------------------------------------------------------------------

def cmd_commit(project_path: Path, message: str) -> int:
    log_path = project_path / _COMMITS_LOG
    ts = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
    with open(log_path, "a", encoding="utf-8") as f:
        f.write(f"{ts} | {message}\n")
    print(f"[{ts}] {message}")
    return 0


def cmd_up(project_path: Path, verbose: bool, dry_run: bool) -> int:
    from .cache import load_server_config

    server = load_server_config()
    if server is None:
        _print_server_error()
        return 1

    # Require a local commit
    log_path = project_path / _COMMITS_LOG
    local_lines = _read_log_lines(log_path)
    if not local_lines:
        print(f"Error: No commits. Run: gnommo -p {project_path.name} commit -m 'message'")
        return 1
    local_last = _last_timestamp(local_lines)

    # Fetch server's commits.log and compare
    remote_project = f"{server['path']}/{project_path.name}"
    ssh_cmd = ["ssh", "-p", server["port"], f"{server['user']}@{server['host']}"]
    result = subprocess.run(
        [*ssh_cmd, f"cat {remote_project}/{_COMMITS_LOG} 2>/dev/null"],
        capture_output=True, text=True,
    )
    server_lines = [l for l in result.stdout.splitlines() if l.strip()]
    server_last = _last_timestamp(server_lines)

    if server_last and server_last > local_last:
        print(f"Error: Server has a more recent commit — pull first.")
        print(f"  Local:  {local_last}")
        print(f"  Server: {server_last}")
        print(f"  Run: gnommo -p {project_path.name} down")
        return 1

    # Build manifest
    manifest = _build_manifest(project_path)
    total = len(manifest)
    print(f"Pushing: {project_path.name}  ({total} files, last commit: {local_last})")
    if verbose:
        for f in manifest:
            print(f"  {f}")

    # Build shared assets manifest
    shared_root, shared_files = _build_shared_manifest(project_path)
    remote_shared = f"{server['path']}/shared_assets"

    if dry_run:
        print("\n[DRY RUN] Would push project files:")
        for f in manifest:
            print(f"  {f}")
        if shared_files:
            print(f"\n[DRY RUN] Would push shared assets ({len(shared_files)} files):")
            for f in shared_files:
                print(f"  {f}")
        return 0

    # Pass 1: project files
    subprocess.run([*ssh_cmd, f"mkdir -p {remote_project}"], check=True)
    rsync_cmd = [
        "rsync", "-av", "--progress",
        "--files-from=-",
        "-e", f"ssh -p {server['port']}",
        f"{project_path}/",
        f"{server['user']}@{server['host']}:{remote_project}/",
    ]
    result = subprocess.run(rsync_cmd, input="\n".join(manifest), text=True)
    if result.returncode != 0:
        print(f"Error: rsync failed for project files (code {result.returncode})")
        return 1

    # Pass 2: shared assets referenced by this project
    if shared_files and shared_root:
        print(f"\nPushing shared assets ({len(shared_files)} files)...")
        subprocess.run([*ssh_cmd, f"mkdir -p {remote_shared}"], check=True)
        rsync_shared = [
            "rsync", "-av", "--progress",
            "--files-from=-",
            "-e", f"ssh -p {server['port']}",
            f"{shared_root}/",
            f"{server['user']}@{server['host']}:{remote_shared}/",
        ]
        result = subprocess.run(rsync_shared, input="\n".join(shared_files), text=True)
        if result.returncode != 0:
            print(f"Error: rsync failed for shared assets (code {result.returncode})")
            return 1

    print(f"\nDone. Pushed {total} project files, {len(shared_files)} shared assets.")
    return 0


def cmd_down(project_path: Path, verbose: bool, dry_run: bool) -> int:
    from .cache import load_server_config

    server = load_server_config()
    if server is None:
        _print_server_error()
        return 1

    remote_project = f"{server['path']}/{project_path.name}"

    # Verify the project exists on the server
    ssh_cmd = ["ssh", "-p", server["port"], f"{server['user']}@{server['host']}"]
    result = subprocess.run(
        [*ssh_cmd, f"test -d {remote_project} && echo ok"],
        capture_output=True, text=True,
    )
    if result.stdout.strip() != "ok":
        print(f"Error: Project not found on server: {remote_project}")
        return 1

    remote_log_result = subprocess.run(
        [*ssh_cmd, f"cat {remote_project}/{_COMMITS_LOG} 2>/dev/null"],
        capture_output=True, text=True,
    )
    server_lines = [l for l in remote_log_result.stdout.splitlines() if l.strip()]
    server_last = _last_timestamp(server_lines)

    print(f"Pulling: {project_path.name}  (server last commit: {server_last or 'none'})")

    project_path.mkdir(parents=True, exist_ok=True)

    rsync_cmd = [
        "rsync", "-av", "--progress",
        "-e", f"ssh -p {server['port']}",
        *[f"--exclude={p}" for p in _DOWN_EXCLUDES],
        f"{server['user']}@{server['host']}:{remote_project}/",
        f"{project_path}/",
    ]

    remote_shared = f"{server['path']}/shared_assets"
    local_shared = project_path.parent / "shared_assets"
    rsync_shared_cmd = [
        "rsync", "-av", "--progress",
        "-e", f"ssh -p {server['port']}",
        *[f"--exclude={p}" for p in _DOWN_EXCLUDES],
        f"{server['user']}@{server['host']}:{remote_shared}/",
        f"{local_shared}/",
    ]

    if dry_run:
        rsync_cmd.insert(1, "--dry-run")
        print("\n[DRY RUN] Would pull project:")
        print(f"  {' '.join(rsync_cmd)}")
        print("\n[DRY RUN] Would pull shared assets:")
        print(f"  {' '.join(rsync_shared_cmd)}")
        return 0

    if verbose:
        print(f"  {' '.join(rsync_cmd)}")

    # Pass 1: project files
    result = subprocess.run(rsync_cmd)
    if result.returncode != 0:
        print(f"Error: rsync failed for project files (code {result.returncode})")
        return 1

    # Pass 2: shared assets (only if server has any)
    check = subprocess.run(
        [*ssh_cmd, f"test -d {remote_shared} && echo ok"],
        capture_output=True, text=True,
    )
    if check.stdout.strip() == "ok":
        print("\nPulling shared assets...")
        local_shared.mkdir(parents=True, exist_ok=True)
        result = subprocess.run(rsync_shared_cmd)
        if result.returncode != 0:
            print(f"Error: rsync failed for shared assets (code {result.returncode})")
            return 1

    print("\nDone.")
    return 0


def _print_server_error():
    print("Error: Server not configured. Add to ~/.gnommo.conf:")
    print("  [server]")
    print("  host = 192.168.1.100")
    print("  user = username")
    print("  path = /gnommo/project")
    print("  port = 22")