Adding cli
This commit is contained in:
+26
-85
@@ -57,10 +57,12 @@ Examples:
|
||||
gnommo -p video1 description Generate YouTube description file
|
||||
gnommo -p video1 transcribe Narration file for timing of slides
|
||||
gnommo -p video1 transcribe --final Transcribe outputted file and generate SRT for YouTube
|
||||
gnommo -p video1 archive Sync project to external cache storage
|
||||
gnommo -p video1 archive --dry-run Preview what would be synced
|
||||
gnommo -p video1 up Upload project files to remote server
|
||||
gnommo -p video1 down Download project files from remote server
|
||||
gnommo -p video1 archive Copy project to connected external drive
|
||||
gnommo -p video1 load Copy project from external drive to local
|
||||
gnommo -p video1 commit -m "msg" Record a commit to commits.log (required before up)
|
||||
gnommo -p video1 up Push manifest files to rendering server
|
||||
gnommo -p video1 up --dry-run Preview which files would be pushed
|
||||
gnommo -p video1 down Pull files from rendering server to local
|
||||
gnommo -p video1 extract-audio --combined Extract audio from narration_combined.mov
|
||||
gnommo -p video1 extract-audio --combined --channel left Extract left channel only
|
||||
gnommo -p video1 extract-audio --segment seg01 Extract from a specific segment
|
||||
@@ -100,6 +102,7 @@ Examples:
|
||||
"description",
|
||||
"archive",
|
||||
"load",
|
||||
"commit",
|
||||
"up",
|
||||
"down",
|
||||
"extract-audio",
|
||||
@@ -226,6 +229,13 @@ Examples:
|
||||
dest="alpha_quality",
|
||||
help="For transcode --processed: HEVC alpha quality 0.0-1.0 (default: 0.75; lower=smaller file)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-m",
|
||||
"--message",
|
||||
type=str,
|
||||
default=None,
|
||||
help="For commit: commit message",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--search",
|
||||
type=str,
|
||||
@@ -322,10 +332,18 @@ Examples:
|
||||
return cmd_archive(project_path, args.verbose, args.dry_run)
|
||||
elif action == "load":
|
||||
return cmd_load(project_path, args.verbose, args.dry_run)
|
||||
elif action == "commit":
|
||||
from .transfer import cmd_commit
|
||||
if not args.message:
|
||||
print("Error: -m 'message' is required for commit.")
|
||||
return 1
|
||||
return cmd_commit(project_path, args.message)
|
||||
elif action == "up":
|
||||
return cmd_sync(project_path, args.verbose, args.dry_run, download=False)
|
||||
from .transfer import cmd_up
|
||||
return cmd_up(project_path, args.verbose, args.dry_run)
|
||||
elif action == "down":
|
||||
return cmd_sync(project_path, args.verbose, args.dry_run, download=True)
|
||||
from .transfer import cmd_down
|
||||
return cmd_down(project_path, args.verbose, args.dry_run)
|
||||
elif action == "extract-audio":
|
||||
return cmd_extract_audio(
|
||||
project_path, args.verbose, args.segment, args.channel, args.combined
|
||||
@@ -4407,7 +4425,8 @@ def cmd_all(
|
||||
return result
|
||||
|
||||
print("\n>>> Step 8/8: Upload\n")
|
||||
return cmd_sync(project_path, verbose, dry_run, download=False)
|
||||
from .transfer import cmd_up
|
||||
return cmd_up(project_path, verbose, dry_run)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -4710,84 +4729,6 @@ def cmd_load(project_path: Path, verbose: bool, dry_run: bool) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync(project_path: Path, verbose: bool, dry_run: bool, download: bool) -> int:
|
||||
"""Sync project files to/from the remote server via rsync over SSH."""
|
||||
from .cache import load_server_config
|
||||
|
||||
server = load_server_config()
|
||||
if server is None:
|
||||
print("Error: Server not configured. Add to ~/.gnommo.conf:")
|
||||
print(" [server]")
|
||||
print(" host = 76.13.144.52")
|
||||
print(" user = root")
|
||||
print(" path = /gnommo/project")
|
||||
return 1
|
||||
|
||||
direction = "Downloading from" if download else "Uploading to"
|
||||
print(f"{direction} server: {project_path.name}")
|
||||
|
||||
remote = f"{server['user']}@{server['host']}:{server['path']}/{project_path.name}/"
|
||||
local = f"{project_path}/"
|
||||
|
||||
if download:
|
||||
src, dest = remote, local
|
||||
else:
|
||||
src, dest = local, remote
|
||||
|
||||
print(f" Source: {src}")
|
||||
print(f" Destination: {dest}")
|
||||
|
||||
# Ensure destination directory exists
|
||||
if not dry_run:
|
||||
if download:
|
||||
project_path.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
remote_dir = f"{server['path']}/{project_path.name}"
|
||||
ssh_cmd = [
|
||||
"ssh",
|
||||
"-p",
|
||||
server["port"],
|
||||
f"{server['user']}@{server['host']}",
|
||||
f"mkdir -p {remote_dir}",
|
||||
]
|
||||
if verbose:
|
||||
print(f" Creating remote dir: {remote_dir}")
|
||||
result = subprocess.run(ssh_cmd)
|
||||
if result.returncode != 0:
|
||||
print(f"Error: could not create remote directory {remote_dir}")
|
||||
return 1
|
||||
|
||||
rsync_cmd = [
|
||||
"rsync",
|
||||
"-av",
|
||||
"--progress",
|
||||
"-e",
|
||||
f"ssh -p {server['port']}",
|
||||
*[f"--exclude={p}" for p in _RSYNC_EXCLUDES],
|
||||
# On upload: delete server-side files that no longer exist locally so
|
||||
# the remote stays an exact mirror of the local project.
|
||||
*(["--delete"] if not download else []),
|
||||
src,
|
||||
dest,
|
||||
]
|
||||
|
||||
if dry_run:
|
||||
rsync_cmd.insert(1, "--dry-run")
|
||||
print("\n [DRY RUN] Would execute:")
|
||||
print(f" {' '.join(rsync_cmd)}")
|
||||
else:
|
||||
print("\n Syncing files...")
|
||||
|
||||
if verbose:
|
||||
print(f" Command: {' '.join(rsync_cmd)}")
|
||||
|
||||
result = subprocess.run(rsync_cmd)
|
||||
if result.returncode != 0:
|
||||
print(f"Error: rsync failed with code {result.returncode}")
|
||||
return 1
|
||||
|
||||
print("\nDone.")
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
"""SSH-based file transfer for gnommo projects (commit / up / down).
|
||||
|
||||
Workflow:
|
||||
gnommo -p video1 commit -m "re-recorded slides 7-15"
|
||||
gnommo -p video1 up # push to rendering server
|
||||
gnommo -p video1 down # pull from rendering server
|
||||
|
||||
Design:
|
||||
- commit appends a timestamped entry to commits.log
|
||||
- up checks server commits.log for newer entry (aborts if found),
|
||||
then rsyncs a manifest of only the files needed to render
|
||||
- down rsyncs everything the server has back to local
|
||||
(server is already clean — it only holds what was pushed)
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
_COMMITS_LOG = "commits.log"
|
||||
|
||||
# Dirs/patterns excluded on down (mirrors what up never pushes).
|
||||
_DOWN_EXCLUDES = [
|
||||
"media/narration/processed/",
|
||||
"media/narration/intermediate/",
|
||||
"media/videos/intermediate/",
|
||||
"media/narration/low/",
|
||||
"media/videos/low/",
|
||||
"media/videos/narration_combined.mov",
|
||||
"**/chunks/",
|
||||
"*.tmp",
|
||||
".DS_Store",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manifest builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_manifest(project_path: Path) -> list[str]:
|
||||
"""Return sorted list of project-relative paths required to render."""
|
||||
files: set[str] = set()
|
||||
|
||||
# Read project.json for path overrides
|
||||
proj_json = project_path / "project.json"
|
||||
project: dict = {}
|
||||
if proj_json.exists():
|
||||
try:
|
||||
project = json.loads(proj_json.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
# Fixed project-root files
|
||||
for fixed in ["project.json", "manuscript.txt", _COMMITS_LOG, "citations.json"]:
|
||||
if (project_path / fixed).exists():
|
||||
files.add(fixed)
|
||||
|
||||
# Manuscript (may be at a non-standard path)
|
||||
manuscript_rel = project.get("manuscript", "manuscript.txt")
|
||||
if (project_path / manuscript_rel).exists():
|
||||
files.add(manuscript_rel)
|
||||
|
||||
# Slide images + slides.json
|
||||
slides_rel = project.get("slides", "media/slides")
|
||||
slides_path = project_path / slides_rel
|
||||
# slides might point directly to slides.json — include its parent dir
|
||||
if slides_path.is_file():
|
||||
files.add(slides_rel)
|
||||
slides_path = slides_path.parent
|
||||
if slides_path.is_dir():
|
||||
for f in slides_path.rglob("*"):
|
||||
if f.is_file():
|
||||
files.add(str(f.relative_to(project_path)))
|
||||
|
||||
# Narration
|
||||
narration_rel = project.get("narration", "media/narration/narration.json")
|
||||
narration_json = project_path / narration_rel
|
||||
narration_dir = narration_json.parent
|
||||
if narration_json.exists():
|
||||
files.add(narration_rel)
|
||||
try:
|
||||
data = json.loads(narration_json.read_text(encoding="utf-8"))
|
||||
for entry in data.values():
|
||||
src = entry.get("source_file")
|
||||
if src:
|
||||
candidate = narration_dir / src
|
||||
if candidate.exists():
|
||||
files.add(str(candidate.relative_to(project_path)))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
# Audio — standard location, non-shared entries only
|
||||
audio_json = project_path / "media" / "audio" / "audio.json"
|
||||
if audio_json.exists():
|
||||
files.add("media/audio/audio.json")
|
||||
try:
|
||||
data = json.loads(audio_json.read_text(encoding="utf-8"))
|
||||
audio_dir = audio_json.parent
|
||||
for entry in data.values():
|
||||
if entry.get("is_shared"):
|
||||
continue
|
||||
src = entry.get("file")
|
||||
if src:
|
||||
candidate = audio_dir / src
|
||||
if candidate.exists():
|
||||
files.add(str(candidate.relative_to(project_path)))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
# Videos — non-shared entries only
|
||||
videos_rel = project.get("videos", "media/videos/videos.json")
|
||||
videos_json = project_path / videos_rel
|
||||
videos_dir = videos_json.parent
|
||||
if videos_json.exists():
|
||||
files.add(videos_rel)
|
||||
try:
|
||||
data = json.loads(videos_json.read_text(encoding="utf-8"))
|
||||
for entry in data.values():
|
||||
if entry.get("is_shared"):
|
||||
continue
|
||||
src = entry.get("source_file")
|
||||
if src:
|
||||
candidate = videos_dir / src
|
||||
if candidate.exists():
|
||||
files.add(str(candidate.relative_to(project_path)))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
return sorted(files)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commit log helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _read_log_lines(path: Path) -> list[str]:
|
||||
if not path.exists():
|
||||
return []
|
||||
return [l for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
|
||||
|
||||
def _last_timestamp(lines: list[str]) -> Optional[str]:
|
||||
for line in reversed(lines):
|
||||
ts = line.split(" | ")[0].strip()
|
||||
if ts:
|
||||
return ts
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cmd_commit(project_path: Path, message: str) -> int:
|
||||
log_path = project_path / _COMMITS_LOG
|
||||
ts = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with open(log_path, "a", encoding="utf-8") as f:
|
||||
f.write(f"{ts} | {message}\n")
|
||||
print(f"[{ts}] {message}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_up(project_path: Path, verbose: bool, dry_run: bool) -> int:
|
||||
from .cache import load_server_config
|
||||
|
||||
server = load_server_config()
|
||||
if server is None:
|
||||
_print_server_error()
|
||||
return 1
|
||||
|
||||
# Require a local commit
|
||||
log_path = project_path / _COMMITS_LOG
|
||||
local_lines = _read_log_lines(log_path)
|
||||
if not local_lines:
|
||||
print(f"Error: No commits. Run: gnommo -p {project_path.name} commit -m 'message'")
|
||||
return 1
|
||||
local_last = _last_timestamp(local_lines)
|
||||
|
||||
# Fetch server's commits.log and compare
|
||||
remote_project = f"{server['path']}/{project_path.name}"
|
||||
ssh_cmd = ["ssh", "-p", server["port"], f"{server['user']}@{server['host']}"]
|
||||
result = subprocess.run(
|
||||
[*ssh_cmd, f"cat {remote_project}/{_COMMITS_LOG} 2>/dev/null"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
server_lines = [l for l in result.stdout.splitlines() if l.strip()]
|
||||
server_last = _last_timestamp(server_lines)
|
||||
|
||||
if server_last and server_last > local_last:
|
||||
print(f"Error: Server has a more recent commit — pull first.")
|
||||
print(f" Local: {local_last}")
|
||||
print(f" Server: {server_last}")
|
||||
print(f" Run: gnommo -p {project_path.name} down")
|
||||
return 1
|
||||
|
||||
# Build manifest
|
||||
manifest = _build_manifest(project_path)
|
||||
total = len(manifest)
|
||||
print(f"Pushing: {project_path.name} ({total} files, last commit: {local_last})")
|
||||
if verbose:
|
||||
for f in manifest:
|
||||
print(f" {f}")
|
||||
|
||||
if dry_run:
|
||||
print("\n[DRY RUN] Would push:")
|
||||
for f in manifest:
|
||||
print(f" {f}")
|
||||
return 0
|
||||
|
||||
# Ensure remote dir exists
|
||||
subprocess.run([*ssh_cmd, f"mkdir -p {remote_project}"], check=True)
|
||||
|
||||
rsync_cmd = [
|
||||
"rsync", "-av", "--progress",
|
||||
"--files-from=-",
|
||||
"-e", f"ssh -p {server['port']}",
|
||||
f"{project_path}/",
|
||||
f"{server['user']}@{server['host']}:{remote_project}/",
|
||||
]
|
||||
result = subprocess.run(rsync_cmd, input="\n".join(manifest), text=True)
|
||||
if result.returncode != 0:
|
||||
print(f"Error: rsync failed (code {result.returncode})")
|
||||
return 1
|
||||
|
||||
print(f"\nDone. Pushed {total} files.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_down(project_path: Path, verbose: bool, dry_run: bool) -> int:
|
||||
from .cache import load_server_config
|
||||
|
||||
server = load_server_config()
|
||||
if server is None:
|
||||
_print_server_error()
|
||||
return 1
|
||||
|
||||
remote_project = f"{server['path']}/{project_path.name}"
|
||||
|
||||
# Verify the project exists on the server
|
||||
ssh_cmd = ["ssh", "-p", server["port"], f"{server['user']}@{server['host']}"]
|
||||
result = subprocess.run(
|
||||
[*ssh_cmd, f"test -d {remote_project} && echo ok"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
if result.stdout.strip() != "ok":
|
||||
print(f"Error: Project not found on server: {remote_project}")
|
||||
return 1
|
||||
|
||||
remote_log_result = subprocess.run(
|
||||
[*ssh_cmd, f"cat {remote_project}/{_COMMITS_LOG} 2>/dev/null"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
server_lines = [l for l in remote_log_result.stdout.splitlines() if l.strip()]
|
||||
server_last = _last_timestamp(server_lines)
|
||||
|
||||
print(f"Pulling: {project_path.name} (server last commit: {server_last or 'none'})")
|
||||
|
||||
project_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
rsync_cmd = [
|
||||
"rsync", "-av", "--progress",
|
||||
"-e", f"ssh -p {server['port']}",
|
||||
*[f"--exclude={p}" for p in _DOWN_EXCLUDES],
|
||||
f"{server['user']}@{server['host']}:{remote_project}/",
|
||||
f"{project_path}/",
|
||||
]
|
||||
|
||||
if dry_run:
|
||||
rsync_cmd.insert(1, "--dry-run")
|
||||
print("\n[DRY RUN] Would execute:")
|
||||
print(f" {' '.join(rsync_cmd)}")
|
||||
return 0
|
||||
|
||||
if verbose:
|
||||
print(f" {' '.join(rsync_cmd)}")
|
||||
|
||||
result = subprocess.run(rsync_cmd)
|
||||
if result.returncode != 0:
|
||||
print(f"Error: rsync failed (code {result.returncode})")
|
||||
return 1
|
||||
|
||||
print("\nDone.")
|
||||
return 0
|
||||
|
||||
|
||||
def _print_server_error():
|
||||
print("Error: Server not configured. Add to ~/.gnommo.conf:")
|
||||
print(" [server]")
|
||||
print(" host = 192.168.1.100")
|
||||
print(" user = username")
|
||||
print(" path = /gnommo/project")
|
||||
print(" port = 22")
|
||||
Reference in New Issue
Block a user