From 5430e1a9c5bd69b982df8926740e86ff68214fc3 Mon Sep 17 00:00:00 2001 From: wangfeng Date: Mon, 4 May 2026 16:27:26 -0700 Subject: [PATCH] feat(skills): remote registry sync with /skills sync command (closes #433) Add `sync_registry` to `skills/install.rs` that pulls `index.json` from the configured `[skills] registry_url`, resolves each entry to a download URL, checks ETag + SHA-256 for freshness, and writes SKILL.md (or the full unpacked tarball) into `~/.deepseek/cache/skills//`. A `.cache-meta.json` sidecar records the ETag and hash so subsequent syncs skip unchanged skills in one round-trip. Wire the new `/skills sync` slash-command in `commands/skills.rs` (dispatched from `list_skills`) and update the `COMMANDS` usage string in `mod.rs` to `/skills [--remote|sync]`. The per-skill failure model is non-fatal: the command prints a per-entry `[+]`/`[=]`/`[!]` summary and returns a final tally. `default_cache_skills_dir` and the new outcome types are re-exported from `skills/mod.rs` for downstream consumers. Co-Authored-By: Claude Sonnet 4.6 --- crates/tui/src/commands/mod.rs | 2 +- crates/tui/src/commands/skills.rs | 81 +++++++- crates/tui/src/skills/install.rs | 326 +++++++++++++++++++++++++++++- crates/tui/src/skills/mod.rs | 2 +- 4 files changed, 406 insertions(+), 5 deletions(-) diff --git a/crates/tui/src/commands/mod.rs b/crates/tui/src/commands/mod.rs index cdfcb3f6..5e67c3b8 100644 --- a/crates/tui/src/commands/mod.rs +++ b/crates/tui/src/commands/mod.rs @@ -409,7 +409,7 @@ pub const COMMANDS: &[CommandInfo] = &[ CommandInfo { name: "skills", aliases: &[], - usage: "/skills [--remote]", + usage: "/skills [--remote|sync]", description_id: MessageId::CmdSkillsDescription, }, CommandInfo { diff --git a/crates/tui/src/commands/skills.rs b/crates/tui/src/commands/skills.rs index 65b264e1..1080cd5b 100644 --- a/crates/tui/src/commands/skills.rs +++ b/crates/tui/src/commands/skills.rs @@ -6,7 +6,7 @@ use crate::network_policy::NetworkPolicy; use crate::skills::SkillRegistry; use crate::skills::install::{ self, DEFAULT_MAX_SIZE_BYTES, DEFAULT_REGISTRY_URL, InstallOutcome, InstallSource, - RegistryFetchResult, UpdateResult, + RegistryFetchResult, SkillSyncOutcome, SyncResult, UpdateResult, }; use crate::tui::app::App; use crate::tui::history::HistoryCell; @@ -28,14 +28,19 @@ fn render_skill_warnings(registry: &SkillRegistry) -> String { /// List all available skills. Pass `--remote` (or `remote`) to fetch the /// curated registry instead of scanning the local skills directory. +/// Pass `sync` to pull the registry index and download all skills to the +/// local cache (`~/.deepseek/cache/skills/`). pub fn list_skills(app: &mut App, arg: Option<&str>) -> CommandResult { if let Some(arg) = arg { let trimmed = arg.trim(); if trimmed == "--remote" || trimmed == "remote" { return list_remote_skills(app); } + if trimmed == "sync" || trimmed == "--sync" { + return sync_skills(app); + } if !trimmed.is_empty() { - return CommandResult::error("Usage: /skills [--remote]"); + return CommandResult::error("Usage: /skills [--remote|sync]"); } } let skills_dir = app.skills_dir.clone(); @@ -284,6 +289,78 @@ pub fn list_remote_skills(app: &mut App) -> CommandResult { } } +// ─── /skills sync ────────────────────────────────────────────────────────── + +/// Fetch the remote registry index and download every listed skill into the +/// local cache (`~/.deepseek/cache/skills//`). +/// +/// For each skill the sync checks the cached ETag / SHA-256 before +/// downloading so unchanged skills are skipped in O(1) network round-trips. +fn sync_skills(app: &mut App) -> CommandResult { + let (network, max_size, registry_url) = installer_settings(app); + let cache_dir = install::default_cache_skills_dir(); + + let result = run_async(async move { + install::sync_registry(&network, ®istry_url, &cache_dir, max_size).await + }); + + match result { + Ok(SyncResult::RegistryDenied(host)) => { + CommandResult::error(network_denied_message(&host)) + } + Ok(SyncResult::RegistryNeedsApproval(host)) => { + CommandResult::error(needs_approval_message(&host)) + } + Ok(SyncResult::Done { outcomes }) => { + let total = outcomes.len(); + let mut downloaded = 0usize; + let mut fresh = 0usize; + let mut failed = 0usize; + let mut out = String::from("Registry sync complete.\n\n"); + + for outcome in &outcomes { + match outcome { + SkillSyncOutcome::Downloaded { name, path } => { + downloaded += 1; + let _ = writeln!( + out, + " [+] {name} — downloaded to {}", + path.display() + ); + } + SkillSyncOutcome::Fresh { name } => { + fresh += 1; + let _ = writeln!(out, " [=] {name} — already up to date"); + } + SkillSyncOutcome::Failed { name, reason } => { + failed += 1; + let _ = writeln!(out, " [!] {name} — failed: {reason}"); + } + SkillSyncOutcome::Denied { name, host } => { + failed += 1; + let _ = writeln!(out, " [x] {name} — network denied ({host})"); + } + SkillSyncOutcome::NeedsApproval { name, host } => { + failed += 1; + let _ = writeln!( + out, + " [?] {name} — needs approval for {host} (run `/network allow {host}` then retry)" + ); + } + } + } + + let _ = write!( + out, + "\n{total} skill(s) processed: {downloaded} downloaded, {fresh} up-to-date, {failed} failed." + ); + + CommandResult::message(out) + } + Err(err) => CommandResult::error(format!("Sync failed: {err:#}")), + } +} + // ─── helpers ─────────────────────────────────────────────────────────────── /// Read the active config knobs for the installer. diff --git a/crates/tui/src/skills/install.rs b/crates/tui/src/skills/install.rs index c67fa856..dc0879a4 100644 --- a/crates/tui/src/skills/install.rs +++ b/crates/tui/src/skills/install.rs @@ -36,12 +36,23 @@ use std::path::{Component, Path, PathBuf}; use anyhow::{Context, Result, bail}; use flate2::read::GzDecoder; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use thiserror::Error; use crate::network_policy::{Decision, NetworkPolicy, host_from_url}; +/// Cache directory for registry-synced skills. +/// +/// Lives at `~/.deepseek/cache/skills/` so it's separate from user-installed +/// skills and can be blown away without losing anything irreplaceable. +pub fn default_cache_skills_dir() -> PathBuf { + dirs::home_dir().map_or_else( + || PathBuf::from("/tmp/deepseek/cache/skills"), + |p| p.join(".deepseek").join("cache").join("skills"), + ) +} + /// Default registry. Falls back to a community-curated `index.json` hosted on /// GitHub raw; users can override via `[skills] registry_url` in config.toml. pub const DEFAULT_REGISTRY_URL: &str = @@ -490,6 +501,319 @@ pub async fn fetch_registry( Ok(RegistryFetchResult::Loaded(parsed)) } +// ───────────────────────────────────────────────────────────────────────────── +// Registry sync (issue #433) +// ───────────────────────────────────────────────────────────────────────────── + +/// Outcome of a single skill entry during [`sync_registry`]. +#[derive(Debug, Clone)] +pub enum SkillSyncOutcome { + /// Skill downloaded and written to the cache directory. + Downloaded { name: String, path: PathBuf }, + /// Cached bytes match the upstream ETag / SHA-256; nothing written. + Fresh { name: String }, + /// Skill download failed; the error is non-fatal so the sync continues. + Failed { name: String, reason: String }, + /// Network policy blocked the download host. + Denied { name: String, host: String }, + /// Network policy requires user approval for the download host. + NeedsApproval { name: String, host: String }, +} + +/// Overall result of [`sync_registry`]. +#[derive(Debug)] +pub enum SyncResult { + /// Sync completed. `outcomes` contains one entry per skill in the index. + Done { outcomes: Vec }, + /// The registry fetch was blocked by network policy. + RegistryDenied(String), + /// The registry fetch requires user approval. + RegistryNeedsApproval(String), +} + +/// Freshness metadata written alongside each cached skill so subsequent syncs +/// can skip unchanged content. +#[derive(Debug, Serialize, Deserialize)] +struct CacheMeta { + /// ETag returned by the server for the primary asset, if any. + #[serde(default)] + etag: Option, + /// SHA-256 hex digest of the downloaded bytes. + sha256: String, + /// Source URL the asset was fetched from. + url: String, +} + +/// Sync the remote registry to the local cache. +/// +/// For every skill listed in `index.json` this function: +/// +/// 1. Resolves the download URL (same logic as `install`). +/// 2. Checks the cached [`CacheMeta`] (etag + sha256) for freshness; skips +/// the download if unchanged. +/// 3. Downloads SKILL.md (and any companion files if the source is a tarball) +/// into `//`. +/// 4. Writes updated [`CacheMeta`] so the next sync is fast. +/// +/// Failures per-skill are non-fatal: [`SkillSyncOutcome::Failed`] is recorded +/// and the sync continues. The caller decides how to surface per-skill errors. +pub async fn sync_registry( + network: &NetworkPolicy, + registry_url: &str, + cache_dir: &Path, + max_size: u64, +) -> Result { + let doc = match fetch_registry(network, registry_url).await? { + RegistryFetchResult::Loaded(doc) => doc, + RegistryFetchResult::Denied(host) => return Ok(SyncResult::RegistryDenied(host)), + RegistryFetchResult::NeedsApproval(host) => { + return Ok(SyncResult::RegistryNeedsApproval(host)); + } + }; + + let mut outcomes = Vec::new(); + + for (name, entry) in &doc.skills { + let outcome = sync_one_skill(name, entry, network, cache_dir, max_size).await; + outcomes.push(outcome); + } + + Ok(SyncResult::Done { outcomes }) +} + +/// Sync a single skill entry from the registry into the cache directory. +async fn sync_one_skill( + name: &str, + entry: &RegistryEntry, + network: &NetworkPolicy, + cache_dir: &Path, + max_size: u64, +) -> SkillSyncOutcome { + // Resolve the source to a concrete URL list. + let source = match InstallSource::parse(&entry.source) { + Ok(s) => s, + Err(err) => { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!("invalid source spec '{}': {err:#}", entry.source), + }; + } + }; + + // Registry sources in index.json must not point back at another registry. + if matches!(source, InstallSource::Registry(_)) { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!( + "registry entry for '{name}' must not point to another registry entry" + ), + }; + } + + let urls = match &source { + InstallSource::GitHubRepo(repo) => vec![ + format!("https://github.com/{repo}/archive/refs/heads/main.tar.gz"), + format!("https://github.com/{repo}/archive/refs/heads/master.tar.gz"), + ], + InstallSource::DirectUrl(url) => vec![url.clone()], + InstallSource::Registry(_) => unreachable!("guarded above"), + }; + + // Check the first downloadable URL against any cached meta. + let skill_cache_dir = cache_dir.join(name); + let meta_path = skill_cache_dir.join(".cache-meta.json"); + + // Try each candidate URL in order. + for url in &urls { + let host = match host_from_url(url) { + Some(h) => h, + None => continue, + }; + match network.decide(&host) { + Decision::Allow => {} + Decision::Deny => { + return SkillSyncOutcome::Denied { + name: name.to_string(), + host, + }; + } + Decision::Prompt => { + return SkillSyncOutcome::NeedsApproval { + name: name.to_string(), + host, + }; + } + } + + // Perform a HEAD request (or conditional GET) for freshness. We use a + // simple GET with If-None-Match when we have an ETag, falling back to + // an unconditional GET for servers that don't support ETags. + let existing_meta: Option = meta_path + .exists() + .then(|| { + fs::read_to_string(&meta_path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + }) + .flatten(); + + // Build the request — add If-None-Match if we have a cached ETag. + let client = reqwest::Client::new(); + let mut req = client.get(url); + if let Some(ref meta) = existing_meta { + if let Some(ref etag) = meta.etag { + req = req.header("If-None-Match", etag); + } + } + + let resp = match req.send().await { + Ok(r) => r, + Err(err) => { + // Network error — try the next candidate URL. + let _ = err; + continue; + } + }; + + let status = resp.status(); + + // 304 Not Modified: cached copy is still fresh. + if status == reqwest::StatusCode::NOT_MODIFIED { + return SkillSyncOutcome::Fresh { + name: name.to_string(), + }; + } + + if status == reqwest::StatusCode::NOT_FOUND { + // Try next URL (main → master fallback). + continue; + } + + if !status.is_success() { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!("GET {url} returned HTTP {status}"), + }; + } + + // Capture ETag before consuming the response body. + let etag = resp + .headers() + .get(reqwest::header::ETAG) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + + let compressed_cap = max_size.saturating_mul(4); + let bytes = match resp.bytes().await { + Ok(b) => b, + Err(err) => { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!("failed to read body from {url}: {err:#}"), + }; + } + }; + if bytes.len() as u64 > compressed_cap { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!( + "download from {url} exceeds compressed size cap ({} bytes)", + compressed_cap + ), + }; + } + + // Compute SHA-256 of the downloaded bytes. + let mut hasher = Sha256::new(); + hasher.update(&bytes); + let sha256 = format!("{:x}", hasher.finalize()); + + // Short-circuit: if the hash matches the cached one, we're fresh even + // without a 304 (some CDNs strip ETags on redirects). + if let Some(ref meta) = existing_meta { + if meta.sha256 == sha256 && meta.url == *url { + return SkillSyncOutcome::Fresh { + name: name.to_string(), + }; + } + } + + // Determine whether this is a tarball or a plain SKILL.md. + // Heuristic: the URL ends with `.tar.gz` or `.tgz`, or the content + // starts with the gzip magic bytes (0x1f 0x8b). + let is_tarball = url.ends_with(".tar.gz") + || url.ends_with(".tgz") + || bytes.starts_with(&[0x1f, 0x8b]); + + let final_path: PathBuf; + + if is_tarball { + // Extract into a temp staging dir, then rename atomically. + let staged = match stage_tarball(&bytes, cache_dir, max_size) { + Ok(s) => s, + Err(err) => { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!("tarball extraction failed: {err:#}"), + }; + } + }; + // Move staged dir into its final location, replacing any prior cache. + let dest = cache_dir.join(name); + if dest.exists() { + let _ = fs::remove_dir_all(&dest); + } + if let Err(err) = fs::rename(&staged.staged_path, &dest) { + let _ = fs::remove_dir_all(&staged.staged_path); + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!("failed to move staged skill into cache: {err:#}"), + }; + } + final_path = dest; + } else { + // Plain SKILL.md (or other companion text file). Write directly. + if let Err(err) = fs::create_dir_all(&skill_cache_dir) { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!("failed to create cache dir: {err:#}"), + }; + } + let skill_md_path = skill_cache_dir.join("SKILL.md"); + if let Err(err) = fs::write(&skill_md_path, &bytes) { + return SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!("failed to write SKILL.md to cache: {err:#}"), + }; + } + final_path = skill_cache_dir.clone(); + } + + // Write the updated freshness metadata. + let meta = CacheMeta { + etag, + sha256, + url: url.clone(), + }; + let meta_json = serde_json::to_string(&meta).unwrap_or_default(); + let _ = fs::write(final_path.join(".cache-meta.json"), meta_json); + + return SkillSyncOutcome::Downloaded { + name: name.to_string(), + path: final_path, + }; + } + + // All candidate URLs exhausted without a successful response. + SkillSyncOutcome::Failed { + name: name.to_string(), + reason: format!( + "all candidate URLs for '{}' failed or were not found", + entry.source + ), + } +} + // ───────────────────────────────────────────────────────────────────────────── // Internal helpers // ───────────────────────────────────────────────────────────────────────────── diff --git a/crates/tui/src/skills/mod.rs b/crates/tui/src/skills/mod.rs index cb1b030f..07b710d2 100644 --- a/crates/tui/src/skills/mod.rs +++ b/crates/tui/src/skills/mod.rs @@ -10,7 +10,7 @@ mod system; pub use install::{ DEFAULT_MAX_SIZE_BYTES, DEFAULT_REGISTRY_URL, INSTALLED_FROM_MARKER, InstallOutcome, InstallSource, InstalledSkill, RegistryDocument, RegistryEntry, RegistryFetchResult, - UpdateResult, + SkillSyncOutcome, SyncResult, UpdateResult, default_cache_skills_dir, }; pub use system::install_system_skills;