feat(skills): remote registry sync with /skills sync command (#654)
This commit is contained in:
Hunter Bown
@@ -409,7 +409,7 @@ pub const COMMANDS: &[CommandInfo] = &[
|
||||
CommandInfo {
|
||||
name: "skills",
|
||||
aliases: &[],
|
||||
usage: "/skills [--remote]",
|
||||
usage: "/skills [--remote|sync]",
|
||||
description_id: MessageId::CmdSkillsDescription,
|
||||
},
|
||||
CommandInfo {
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::network_policy::NetworkPolicy;
|
||||
use crate::skills::SkillRegistry;
|
||||
use crate::skills::install::{
|
||||
self, DEFAULT_MAX_SIZE_BYTES, DEFAULT_REGISTRY_URL, InstallOutcome, InstallSource,
|
||||
RegistryFetchResult, UpdateResult,
|
||||
RegistryFetchResult, SkillSyncOutcome, SyncResult, UpdateResult,
|
||||
};
|
||||
use crate::tui::app::App;
|
||||
use crate::tui::history::HistoryCell;
|
||||
@@ -28,14 +28,19 @@ fn render_skill_warnings(registry: &SkillRegistry) -> String {
|
||||
|
||||
/// List all available skills. Pass `--remote` (or `remote`) to fetch the
|
||||
/// curated registry instead of scanning the local skills directory.
|
||||
/// Pass `sync` to pull the registry index and download all skills to the
|
||||
/// local cache (`~/.deepseek/cache/skills/`).
|
||||
pub fn list_skills(app: &mut App, arg: Option<&str>) -> CommandResult {
|
||||
if let Some(arg) = arg {
|
||||
let trimmed = arg.trim();
|
||||
if trimmed == "--remote" || trimmed == "remote" {
|
||||
return list_remote_skills(app);
|
||||
}
|
||||
if trimmed == "sync" || trimmed == "--sync" {
|
||||
return sync_skills(app);
|
||||
}
|
||||
if !trimmed.is_empty() {
|
||||
return CommandResult::error("Usage: /skills [--remote]");
|
||||
return CommandResult::error("Usage: /skills [--remote|sync]");
|
||||
}
|
||||
}
|
||||
let skills_dir = app.skills_dir.clone();
|
||||
@@ -296,6 +301,78 @@ pub fn list_remote_skills(app: &mut App) -> CommandResult {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── /skills sync ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Fetch the remote registry index and download every listed skill into the
|
||||
/// local cache (`~/.deepseek/cache/skills/<name>/`).
|
||||
///
|
||||
/// For each skill the sync checks the cached ETag / SHA-256 before
|
||||
/// downloading so unchanged skills are skipped in O(1) network round-trips.
|
||||
fn sync_skills(app: &mut App) -> CommandResult {
|
||||
let (network, max_size, registry_url) = installer_settings(app);
|
||||
let cache_dir = install::default_cache_skills_dir();
|
||||
|
||||
let result = run_async(async move {
|
||||
install::sync_registry(&network, ®istry_url, &cache_dir, max_size).await
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(SyncResult::RegistryDenied(host)) => {
|
||||
CommandResult::error(network_denied_message(&host))
|
||||
}
|
||||
Ok(SyncResult::RegistryNeedsApproval(host)) => {
|
||||
CommandResult::error(needs_approval_message(&host))
|
||||
}
|
||||
Ok(SyncResult::Done { outcomes }) => {
|
||||
let total = outcomes.len();
|
||||
let mut downloaded = 0usize;
|
||||
let mut fresh = 0usize;
|
||||
let mut failed = 0usize;
|
||||
let mut out = String::from("Registry sync complete.\n\n");
|
||||
|
||||
for outcome in &outcomes {
|
||||
match outcome {
|
||||
SkillSyncOutcome::Downloaded { name, path } => {
|
||||
downloaded += 1;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" [+] {name} — downloaded to {}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
SkillSyncOutcome::Fresh { name } => {
|
||||
fresh += 1;
|
||||
let _ = writeln!(out, " [=] {name} — already up to date");
|
||||
}
|
||||
SkillSyncOutcome::Failed { name, reason } => {
|
||||
failed += 1;
|
||||
let _ = writeln!(out, " [!] {name} — failed: {reason}");
|
||||
}
|
||||
SkillSyncOutcome::Denied { name, host } => {
|
||||
failed += 1;
|
||||
let _ = writeln!(out, " [x] {name} — network denied ({host})");
|
||||
}
|
||||
SkillSyncOutcome::NeedsApproval { name, host } => {
|
||||
failed += 1;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" [?] {name} — needs approval for {host} (run `/network allow {host}` then retry)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = write!(
|
||||
out,
|
||||
"\n{total} skill(s) processed: {downloaded} downloaded, {fresh} up-to-date, {failed} failed."
|
||||
);
|
||||
|
||||
CommandResult::message(out)
|
||||
}
|
||||
Err(err) => CommandResult::error(format!("Sync failed: {err:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
/// Read the active config knobs for the installer.
|
||||
|
||||
@@ -36,12 +36,23 @@ use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use flate2::read::GzDecoder;
|
||||
use serde::Deserialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::network_policy::{Decision, NetworkPolicy, host_from_url};
|
||||
|
||||
/// Cache directory for registry-synced skills.
|
||||
///
|
||||
/// Lives at `~/.deepseek/cache/skills/` so it's separate from user-installed
|
||||
/// skills and can be blown away without losing anything irreplaceable.
|
||||
pub fn default_cache_skills_dir() -> PathBuf {
|
||||
dirs::home_dir().map_or_else(
|
||||
|| PathBuf::from("/tmp/deepseek/cache/skills"),
|
||||
|p| p.join(".deepseek").join("cache").join("skills"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Default registry. Falls back to a community-curated `index.json` hosted on
|
||||
/// GitHub raw; users can override via `[skills] registry_url` in config.toml.
|
||||
pub const DEFAULT_REGISTRY_URL: &str =
|
||||
@@ -490,6 +501,319 @@ pub async fn fetch_registry(
|
||||
Ok(RegistryFetchResult::Loaded(parsed))
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Registry sync (issue #433)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Outcome of a single skill entry during [`sync_registry`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum SkillSyncOutcome {
|
||||
/// Skill downloaded and written to the cache directory.
|
||||
Downloaded { name: String, path: PathBuf },
|
||||
/// Cached bytes match the upstream ETag / SHA-256; nothing written.
|
||||
Fresh { name: String },
|
||||
/// Skill download failed; the error is non-fatal so the sync continues.
|
||||
Failed { name: String, reason: String },
|
||||
/// Network policy blocked the download host.
|
||||
Denied { name: String, host: String },
|
||||
/// Network policy requires user approval for the download host.
|
||||
NeedsApproval { name: String, host: String },
|
||||
}
|
||||
|
||||
/// Overall result of [`sync_registry`].
|
||||
#[derive(Debug)]
|
||||
pub enum SyncResult {
|
||||
/// Sync completed. `outcomes` contains one entry per skill in the index.
|
||||
Done { outcomes: Vec<SkillSyncOutcome> },
|
||||
/// The registry fetch was blocked by network policy.
|
||||
RegistryDenied(String),
|
||||
/// The registry fetch requires user approval.
|
||||
RegistryNeedsApproval(String),
|
||||
}
|
||||
|
||||
/// Freshness metadata written alongside each cached skill so subsequent syncs
|
||||
/// can skip unchanged content.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct CacheMeta {
|
||||
/// ETag returned by the server for the primary asset, if any.
|
||||
#[serde(default)]
|
||||
etag: Option<String>,
|
||||
/// SHA-256 hex digest of the downloaded bytes.
|
||||
sha256: String,
|
||||
/// Source URL the asset was fetched from.
|
||||
url: String,
|
||||
}
|
||||
|
||||
/// Sync the remote registry to the local cache.
|
||||
///
|
||||
/// For every skill listed in `index.json` this function:
|
||||
///
|
||||
/// 1. Resolves the download URL (same logic as `install`).
|
||||
/// 2. Checks the cached [`CacheMeta`] (etag + sha256) for freshness; skips
|
||||
/// the download if unchanged.
|
||||
/// 3. Downloads SKILL.md (and any companion files if the source is a tarball)
|
||||
/// into `<cache_dir>/<name>/`.
|
||||
/// 4. Writes updated [`CacheMeta`] so the next sync is fast.
|
||||
///
|
||||
/// Failures per-skill are non-fatal: [`SkillSyncOutcome::Failed`] is recorded
|
||||
/// and the sync continues. The caller decides how to surface per-skill errors.
|
||||
pub async fn sync_registry(
|
||||
network: &NetworkPolicy,
|
||||
registry_url: &str,
|
||||
cache_dir: &Path,
|
||||
max_size: u64,
|
||||
) -> Result<SyncResult> {
|
||||
let doc = match fetch_registry(network, registry_url).await? {
|
||||
RegistryFetchResult::Loaded(doc) => doc,
|
||||
RegistryFetchResult::Denied(host) => return Ok(SyncResult::RegistryDenied(host)),
|
||||
RegistryFetchResult::NeedsApproval(host) => {
|
||||
return Ok(SyncResult::RegistryNeedsApproval(host));
|
||||
}
|
||||
};
|
||||
|
||||
let mut outcomes = Vec::new();
|
||||
|
||||
for (name, entry) in &doc.skills {
|
||||
let outcome = sync_one_skill(name, entry, network, cache_dir, max_size).await;
|
||||
outcomes.push(outcome);
|
||||
}
|
||||
|
||||
Ok(SyncResult::Done { outcomes })
|
||||
}
|
||||
|
||||
/// Sync a single skill entry from the registry into the cache directory.
|
||||
async fn sync_one_skill(
|
||||
name: &str,
|
||||
entry: &RegistryEntry,
|
||||
network: &NetworkPolicy,
|
||||
cache_dir: &Path,
|
||||
max_size: u64,
|
||||
) -> SkillSyncOutcome {
|
||||
// Resolve the source to a concrete URL list.
|
||||
let source = match InstallSource::parse(&entry.source) {
|
||||
Ok(s) => s,
|
||||
Err(err) => {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!("invalid source spec '{}': {err:#}", entry.source),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Registry sources in index.json must not point back at another registry.
|
||||
if matches!(source, InstallSource::Registry(_)) {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!(
|
||||
"registry entry for '{name}' must not point to another registry entry"
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
let urls = match &source {
|
||||
InstallSource::GitHubRepo(repo) => vec![
|
||||
format!("https://github.com/{repo}/archive/refs/heads/main.tar.gz"),
|
||||
format!("https://github.com/{repo}/archive/refs/heads/master.tar.gz"),
|
||||
],
|
||||
InstallSource::DirectUrl(url) => vec![url.clone()],
|
||||
InstallSource::Registry(_) => unreachable!("guarded above"),
|
||||
};
|
||||
|
||||
// Check the first downloadable URL against any cached meta.
|
||||
let skill_cache_dir = cache_dir.join(name);
|
||||
let meta_path = skill_cache_dir.join(".cache-meta.json");
|
||||
|
||||
// Try each candidate URL in order.
|
||||
for url in &urls {
|
||||
let host = match host_from_url(url) {
|
||||
Some(h) => h,
|
||||
None => continue,
|
||||
};
|
||||
match network.decide(&host) {
|
||||
Decision::Allow => {}
|
||||
Decision::Deny => {
|
||||
return SkillSyncOutcome::Denied {
|
||||
name: name.to_string(),
|
||||
host,
|
||||
};
|
||||
}
|
||||
Decision::Prompt => {
|
||||
return SkillSyncOutcome::NeedsApproval {
|
||||
name: name.to_string(),
|
||||
host,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Perform a HEAD request (or conditional GET) for freshness. We use a
|
||||
// simple GET with If-None-Match when we have an ETag, falling back to
|
||||
// an unconditional GET for servers that don't support ETags.
|
||||
let existing_meta: Option<CacheMeta> = meta_path
|
||||
.exists()
|
||||
.then(|| {
|
||||
fs::read_to_string(&meta_path)
|
||||
.ok()
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
})
|
||||
.flatten();
|
||||
|
||||
// Build the request — add If-None-Match if we have a cached ETag.
|
||||
let client = reqwest::Client::new();
|
||||
let mut req = client.get(url);
|
||||
if let Some(ref meta) = existing_meta {
|
||||
if let Some(ref etag) = meta.etag {
|
||||
req = req.header("If-None-Match", etag);
|
||||
}
|
||||
}
|
||||
|
||||
let resp = match req.send().await {
|
||||
Ok(r) => r,
|
||||
Err(err) => {
|
||||
// Network error — try the next candidate URL.
|
||||
let _ = err;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
// 304 Not Modified: cached copy is still fresh.
|
||||
if status == reqwest::StatusCode::NOT_MODIFIED {
|
||||
return SkillSyncOutcome::Fresh {
|
||||
name: name.to_string(),
|
||||
};
|
||||
}
|
||||
|
||||
if status == reqwest::StatusCode::NOT_FOUND {
|
||||
// Try next URL (main → master fallback).
|
||||
continue;
|
||||
}
|
||||
|
||||
if !status.is_success() {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!("GET {url} returned HTTP {status}"),
|
||||
};
|
||||
}
|
||||
|
||||
// Capture ETag before consuming the response body.
|
||||
let etag = resp
|
||||
.headers()
|
||||
.get(reqwest::header::ETAG)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let compressed_cap = max_size.saturating_mul(4);
|
||||
let bytes = match resp.bytes().await {
|
||||
Ok(b) => b,
|
||||
Err(err) => {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!("failed to read body from {url}: {err:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
if bytes.len() as u64 > compressed_cap {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!(
|
||||
"download from {url} exceeds compressed size cap ({} bytes)",
|
||||
compressed_cap
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
// Compute SHA-256 of the downloaded bytes.
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bytes);
|
||||
let sha256 = format!("{:x}", hasher.finalize());
|
||||
|
||||
// Short-circuit: if the hash matches the cached one, we're fresh even
|
||||
// without a 304 (some CDNs strip ETags on redirects).
|
||||
if let Some(ref meta) = existing_meta {
|
||||
if meta.sha256 == sha256 && meta.url == *url {
|
||||
return SkillSyncOutcome::Fresh {
|
||||
name: name.to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Determine whether this is a tarball or a plain SKILL.md.
|
||||
// Heuristic: the URL ends with `.tar.gz` or `.tgz`, or the content
|
||||
// starts with the gzip magic bytes (0x1f 0x8b).
|
||||
let is_tarball = url.ends_with(".tar.gz")
|
||||
|| url.ends_with(".tgz")
|
||||
|| bytes.starts_with(&[0x1f, 0x8b]);
|
||||
|
||||
let final_path: PathBuf;
|
||||
|
||||
if is_tarball {
|
||||
// Extract into a temp staging dir, then rename atomically.
|
||||
let staged = match stage_tarball(&bytes, cache_dir, max_size) {
|
||||
Ok(s) => s,
|
||||
Err(err) => {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!("tarball extraction failed: {err:#}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
// Move staged dir into its final location, replacing any prior cache.
|
||||
let dest = cache_dir.join(name);
|
||||
if dest.exists() {
|
||||
let _ = fs::remove_dir_all(&dest);
|
||||
}
|
||||
if let Err(err) = fs::rename(&staged.staged_path, &dest) {
|
||||
let _ = fs::remove_dir_all(&staged.staged_path);
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!("failed to move staged skill into cache: {err:#}"),
|
||||
};
|
||||
}
|
||||
final_path = dest;
|
||||
} else {
|
||||
// Plain SKILL.md (or other companion text file). Write directly.
|
||||
if let Err(err) = fs::create_dir_all(&skill_cache_dir) {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!("failed to create cache dir: {err:#}"),
|
||||
};
|
||||
}
|
||||
let skill_md_path = skill_cache_dir.join("SKILL.md");
|
||||
if let Err(err) = fs::write(&skill_md_path, &bytes) {
|
||||
return SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!("failed to write SKILL.md to cache: {err:#}"),
|
||||
};
|
||||
}
|
||||
final_path = skill_cache_dir.clone();
|
||||
}
|
||||
|
||||
// Write the updated freshness metadata.
|
||||
let meta = CacheMeta {
|
||||
etag,
|
||||
sha256,
|
||||
url: url.clone(),
|
||||
};
|
||||
let meta_json = serde_json::to_string(&meta).unwrap_or_default();
|
||||
let _ = fs::write(final_path.join(".cache-meta.json"), meta_json);
|
||||
|
||||
return SkillSyncOutcome::Downloaded {
|
||||
name: name.to_string(),
|
||||
path: final_path,
|
||||
};
|
||||
}
|
||||
|
||||
// All candidate URLs exhausted without a successful response.
|
||||
SkillSyncOutcome::Failed {
|
||||
name: name.to_string(),
|
||||
reason: format!(
|
||||
"all candidate URLs for '{}' failed or were not found",
|
||||
entry.source
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Internal helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -10,7 +10,7 @@ mod system;
|
||||
pub use install::{
|
||||
DEFAULT_MAX_SIZE_BYTES, DEFAULT_REGISTRY_URL, INSTALLED_FROM_MARKER, InstallOutcome,
|
||||
InstallSource, InstalledSkill, RegistryDocument, RegistryEntry, RegistryFetchResult,
|
||||
UpdateResult,
|
||||
SkillSyncOutcome, SyncResult, UpdateResult, default_cache_skills_dir,
|
||||
};
|
||||
pub use system::install_system_skills;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user