dcosnet
/
fester
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
fester/backend/storage/cas_api.py

582 lines
20 KiB
Python
Executable File
Raw Blame History

"""
Content-Addressable Store (CAS) — shared artifact cache for sorcery-go + Fester.
This module provides both the storage backend (SQLite-backed CAS table) and
the FastAPI router for HTTP access. sorcery-go pushes build artifacts (including
.svb Sovereign Bundles) here after BundleSovereign completes, and Fester's
PipelineEngine checks the CAS before dispatching — eliminating redundant
rebuilds across the cluster.
Storage layout:
- /var/lib/fester/cas/objects/<sha256[0:2]>/<sha256[2:]> — artifact file
- /var/lib/fester/cas/meta/<sha256>.json — metadata JSON
API endpoints:
PUT /api/cas/{sha256} — store an artifact (multipart upload)
GET /api/cas/{sha256} — retrieve an artifact (streamed download)
HEAD /api/cas/{sha256} — check existence (returns metadata JSON)
DELETE /api/cas/{sha256} — remove an artifact
GET /api/cas/ — list all artifacts (paginated)
GET /api/cas/stats — cache statistics (hits, misses, size)
DAG-aware cache integration:
The PipelineEngine calls cas_check(action_hash) before executing each
action. If the artifact exists in the CAS, the action is skipped and
a cache_update event is emitted. This turns a 45-minute dependency
tree into a 2-minute cache assembly.
Cross-runtime deduplication:
An artifact built inside LXC on one node is instantly available to a
Firecracker microVM on another. The CAS key is the content SHA-256,
so the runtime doesn't matter — only the content hash matters.
Integration with sorcery-go:
sorcery-go's BundleSovereign produces .svb files with SHA-256 in the
METADATA.json. The Cauldron's post-bundle hook calls PUT /api/cas/{sha}
with the .svb file. Fester's DAG executor calls HEAD /api/cas/{sha} to
skip already-built targets.
Config (config.yaml or env vars):
cas.enabled: true/false (default: true)
cas.path: /var/lib/fester/cas (or FESTER_CAS_PATH env)
cas.max_size: 50GB (or FESTER_CAS_MAX_SIZE env)
cas.on_btrfs: auto-detect (prefer reflink)
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import sqlite3
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from fastapi import APIRouter, HTTPException, UploadFile, File, Response
from fastapi.responses import FileResponse, JSONResponse
# ----------------------------------------------------------------
# Configuration
# ----------------------------------------------------------------
def _resolve_cas_path() -> Path:
"""Resolve the CAS root directory with graceful fallbacks."""
env = os.environ.get("FESTER_CAS_PATH")
if env:
p = Path(env)
p.mkdir(parents=True, exist_ok=True)
return p
for c in [
Path("/var/lib/fester/cas"),
Path.home() / ".fester" / "cas",
Path("/tmp/fester_cas"),
]:
try:
c.mkdir(parents=True, exist_ok=True)
# Write-test
test = c / ".fester_cas_test"
test.write_text("")
test.unlink()
return c
except (OSError, PermissionError):
continue
return Path("/tmp/fester_cas") # last resort
CAS_ROOT = _resolve_cas_path()
OBJECTS_DIR = CAS_ROOT / "objects"
META_DIR = CAS_ROOT / "meta"
OBJECTS_DIR.mkdir(parents=True, exist_ok=True)
META_DIR.mkdir(parents=True, exist_ok=True)
# Max cache size in bytes (default: 50 GB)
MAX_CAS_SIZE = int(os.environ.get("FESTER_CAS_MAX_SIZE", str(50 * 1024 ** 3)))
# ----------------------------------------------------------------
# Storage backend
# ----------------------------------------------------------------
class CASStore:
"""Content-addressable artifact store.
Artifacts are stored on disk at:
objects/<sha256[0:2]>/<sha256[2:]>
Metadata is stored at:
meta/<sha256>.json
An in-process SQLite table tracks existence, size, and access time
for fast listing and LRU eviction.
"""
def __init__(self, root: Path = CAS_ROOT):
self.root = root
self.objects_dir = root / "objects"
self.meta_dir = root / "meta"
self.objects_dir.mkdir(parents=True, exist_ok=True)
self.meta_dir.mkdir(parents=True, exist_ok=True)
# SQLite index for fast queries
self._db_path = root / "cas.db"
self._lock = threading.Lock()
self._conn = sqlite3.connect(str(self._db_path), check_same_thread=False)
self._conn.row_factory = sqlite3.Row
self._conn.execute("PRAGMA journal_mode=WAL")
self._migrate()
# Cache statistics
self._stats_lock = threading.Lock()
self._stats = {"hits": 0, "misses": 0, "stores": 0}
def _migrate(self):
with self._lock:
self._conn.executescript("""
CREATE TABLE IF NOT EXISTS cas_entries (
sha256 TEXT PRIMARY KEY,
size INTEGER NOT NULL,
content_type TEXT,
source TEXT,
build_id TEXT,
target TEXT,
runtime TEXT,
node TEXT,
created_at REAL,
last_accessed REAL,
access_count INTEGER DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_cas_target ON cas_entries(target);
CREATE INDEX IF NOT EXISTS idx_cas_node ON cas_entries(node);
CREATE INDEX IF NOT EXISTS idx_cas_build ON cas_entries(build_id);
CREATE INDEX IF NOT EXISTS idx_cas_accessed ON cas_entries(last_accessed);
""")
self._conn.commit()
def _object_path(self, sha256: str) -> Path:
"""Return the on-disk path for an artifact."""
prefix = sha256[:2]
suffix = sha256[2:]
return self.objects_dir / prefix / suffix
def _meta_path(self, sha256: str) -> Path:
"""Return the metadata JSON path."""
return self.meta_dir / f"{sha256}.json"
# ----------------------------------------------------------------
# Core operations
# ----------------------------------------------------------------
def exists(self, sha256: str) -> bool:
"""Check if an artifact exists. Updates access time on hit."""
path = self._object_path(sha256)
if not path.exists():
with self._stats_lock:
self._stats["misses"] += 1
return False
with self._stats_lock:
self._stats["hits"] += 1
# Update access time
with self._lock:
self._conn.execute(
"UPDATE cas_entries SET last_accessed = ?, access_count = access_count + 1 WHERE sha256 = ?",
(time.time(), sha256),
)
self._conn.commit()
return True
def store(self, sha256: str, data: bytes,
content_type: str = "application/octet-stream",
source: str = "", build_id: str = "",
target: str = "", runtime: str = "",
node: str = "") -> Dict[str, Any]:
"""Store an artifact. Returns metadata dict."""
path = self._object_path(sha256)
path.parent.mkdir(parents=True, exist_ok=True)
# Check btrfs reflink first — if we already have this hash, we
# can skip the write entirely (content-addressable means no dupes).
if path.exists():
# Already stored — just update metadata.
with self._stats_lock:
self._stats["hits"] += 1
return self.get_meta(sha256)
# Write the artifact
tmp_path = path.parent / f".tmp_{sha256}"
try:
with open(tmp_path, "wb") as f:
f.write(data)
# Atomic rename
tmp_path.rename(path)
except Exception:
if tmp_path.exists():
tmp_path.unlink()
raise
# Write metadata JSON
now = time.time()
meta = {
"sha256": sha256,
"size": len(data),
"content_type": content_type,
"source": source,
"build_id": build_id,
"target": target,
"runtime": runtime,
"node": node,
"created_at": now,
"last_accessed": now,
"access_count": 0,
}
meta_path = self._meta_path(sha256)
with open(meta_path, "w") as f:
json.dump(meta, f, indent=2)
# Update SQLite index
with self._lock:
self._conn.execute("""
INSERT OR REPLACE INTO cas_entries
(sha256, size, content_type, source, build_id, target,
runtime, node, created_at, last_accessed, access_count)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)
""", (sha256, len(data), content_type, source, build_id,
target, runtime, node, now, now))
self._conn.commit()
with self._stats_lock:
self._stats["stores"] += 1
return meta
def store_file(self, sha256: str, src_path: str,
content_type: str = "application/octet-stream",
source: str = "", build_id: str = "",
target: str = "", runtime: str = "",
node: str = "") -> Dict[str, Any]:
"""Store an artifact from a file path (uses reflink on btrfs)."""
path = self._object_path(sha256)
if path.exists():
with self._stats_lock:
self._stats["hits"] += 1
return self.get_meta(sha256)
path.parent.mkdir(parents=True, exist_ok=True)
# Try btrfs reflink first (instant, zero-copy)
import subprocess
try:
result = subprocess.run(
["cp", "--reflink=auto", src_path, str(path)],
capture_output=True, timeout=30,
)
if result.returncode == 0:
size = os.path.getsize(path)
else:
raise OSError(f"cp --reflink failed: {result.stderr.decode()}")
except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
# Fallback: shutil copy
shutil.copy2(src_path, str(path))
size = os.path.getsize(path)
now = time.time()
meta = {
"sha256": sha256,
"size": size,
"content_type": content_type,
"source": source,
"build_id": build_id,
"target": target,
"runtime": runtime,
"node": node,
"created_at": now,
"last_accessed": now,
"access_count": 0,
}
meta_path = self._meta_path(sha256)
with open(meta_path, "w") as f:
json.dump(meta, f, indent=2)
with self._lock:
self._conn.execute("""
INSERT OR REPLACE INTO cas_entries
(sha256, size, content_type, source, build_id, target,
runtime, node, created_at, last_accessed, access_count)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)
""", (sha256, size, content_type, source, build_id,
target, runtime, node, now, now))
self._conn.commit()
with self._stats_lock:
self._stats["stores"] += 1
return meta
def retrieve(self, sha256: str) -> Optional[Tuple[bytes, Dict[str, Any]]]:
"""Retrieve an artifact and its metadata. Returns (data, meta) or None."""
path = self._object_path(sha256)
if not path.exists():
return None
with self._lock:
self._conn.execute(
"UPDATE cas_entries SET last_accessed = ?, access_count = access_count + 1 WHERE sha256 = ?",
(time.time(), sha256),
)
self._conn.commit()
with open(path, "rb") as f:
data = f.read()
meta = self.get_meta(sha256)
return (data, meta)
def retrieve_path(self, sha256: str, dst_path: str) -> bool:
"""Retrieve an artifact to a destination path (uses reflink on btrfs)."""
path = self._object_path(sha256)
if not path.exists():
return False
dst = Path(dst_path)
dst.parent.mkdir(parents=True, exist_ok=True)
# Try reflink
import subprocess
try:
result = subprocess.run(
["cp", "--reflink=auto", str(path), str(dst)],
capture_output=True, timeout=30,
)
if result.returncode == 0:
with self._lock:
self._conn.execute(
"UPDATE cas_entries SET last_accessed = ?, access_count = access_count + 1 WHERE sha256 = ?",
(time.time(), sha256),
)
self._conn.commit()
return True
except (FileNotFoundError, subprocess.TimeoutExpired):
pass
# Fallback
shutil.copy2(str(path), str(dst))
with self._lock:
self._conn.execute(
"UPDATE cas_entries SET last_accessed = ?, access_count = access_count + 1 WHERE sha256 = ?",
(time.time(), sha256),
)
self._conn.commit()
return True
def delete(self, sha256: str) -> bool:
"""Delete an artifact from the CAS."""
path = self._object_path(sha256)
meta_path = self._meta_path(sha256)
removed = False
if path.exists():
path.unlink()
removed = True
if meta_path.exists():
meta_path.unlink()
with self._lock:
self._conn.execute("DELETE FROM cas_entries WHERE sha256 = ?", (sha256,))
self._conn.commit()
return removed
def get_meta(self, sha256: str) -> Optional[Dict[str, Any]]:
"""Get metadata for an artifact."""
meta_path = self._meta_path(sha256)
if meta_path.exists():
try:
with open(meta_path) as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
pass
return None
def list_entries(self, limit: int = 100, offset: int = 0,
target: Optional[str] = None) -> List[Dict[str, Any]]:
"""List CAS entries, optionally filtered by target."""
with self._lock:
if target:
rows = self._conn.execute(
"SELECT * FROM cas_entries WHERE target = ? ORDER BY created_at DESC LIMIT ? OFFSET ?",
(target, limit, offset),
).fetchall()
else:
rows = self._conn.execute(
"SELECT * FROM cas_entries ORDER BY created_at DESC LIMIT ? OFFSET ?",
(limit, offset),
).fetchall()
return [dict(r) for r in rows]
def stats(self) -> Dict[str, Any]:
"""Return cache statistics."""
with self._lock:
row = self._conn.execute(
"SELECT COUNT(*) as count, COALESCE(SUM(size), 0) as total_size FROM cas_entries"
).fetchone()
with self._stats_lock:
s = dict(self._stats)
return {
"total_artifacts": row["count"],
"total_bytes": row["total_size"],
"max_bytes": MAX_CAS_SIZE,
"utilization_pct": round(row["total_size"] / MAX_CAS_SIZE * 100, 1) if MAX_CAS_SIZE > 0 else 0,
"hits": s["hits"],
"misses": s["misses"],
"stores": s["stores"],
"hit_rate_pct": round(
s["hits"] / max(s["hits"] + s["misses"], 1) * 100, 1
),
}
def evict_lru(self, target_bytes: int) -> int:
"""Evict least-recently-used entries until we free target_bytes.
Returns the number of entries evicted."""
evicted = 0
freed = 0
with self._lock:
rows = self._conn.execute(
"SELECT sha256, size FROM cas_entries ORDER BY last_accessed ASC"
).fetchall()
for row in rows:
if freed >= target_bytes:
break
sha = row["sha256"]
if self.delete(sha):
freed += row["size"]
evicted += 1
return evicted
def check_action(self, action_hash: str) -> Optional[Dict[str, Any]]:
"""Check if a build action's output is already cached.
This is the DAG-aware cache integration point. The PipelineEngine
calls this before executing each action. If the artifact exists,
the action is skipped entirely.
Args:
action_hash: SHA-256 of the action's expected output.
This can be the content hash of the build output,
or a synthetic hash of (target + env + source_hash).
Returns:
Metadata dict if cached, None if not.
"""
if self.exists(action_hash):
return self.get_meta(action_hash)
return None
def close(self):
with self._lock:
self._conn.close()
# ----------------------------------------------------------------
# Process-wide singleton
# ----------------------------------------------------------------
STORE = CASStore()
# ----------------------------------------------------------------
# FastAPI router
# ----------------------------------------------------------------
router = APIRouter(prefix="/api/cas", tags=["cas"])
@router.get("/stats")
async def api_cas_stats():
"""Cache statistics — hits, misses, utilization."""
return STORE.stats()
@router.get("")
async def api_cas_list(limit: int = 100, offset: int = 0, target: Optional[str] = None):
"""List all cached artifacts."""
return {"artifacts": STORE.list_entries(limit=limit, offset=offset, target=target)}
@router.head("/{sha256}")
async def api_cas_head(sha256: str):
"""Check if an artifact exists. Returns metadata JSON on hit."""
if not STORE.exists(sha256):
raise HTTPException(404, "artifact not found")
meta = STORE.get_meta(sha256)
return JSONResponse(content=meta or {"sha256": sha256, "cached": True})
@router.get("/{sha256}")
async def api_cas_get(sha256: str):
"""Retrieve an artifact. Streams the file from disk."""
path = STORE._object_path(sha256)
if not path.exists():
raise HTTPException(404, "artifact not found")
meta = STORE.get_meta(sha256)
content_type = (meta or {}).get("content_type", "application/octet-stream")
return FileResponse(
str(path),
media_type=content_type,
filename=f"{sha256[:16]}{'.' + (meta or {}).get('source', '').rsplit('.', 1)[-1] if meta and meta.get('source') else ''}",
)
@router.put("/{sha256}")
async def api_cas_put(
sha256: str,
file: UploadFile = File(...),
source: str = "",
build_id: str = "",
target: str = "",
runtime: str = "",
node: str = "",
):
"""Store an artifact. The sha256 in the URL must match the file content.
Query params:
source: source file name (e.g., "busybox-x86_64.svb")
build_id: sorcery-go or Fester build ID
target: build target (e.g., "x86_64-linux-gnu")
runtime: execution runtime (e.g., "podman", "firecracker")
node: node name that produced the artifact
"""
data = await file.read()
# Verify the SHA-256 matches
actual_sha = hashlib.sha256(data).hexdigest()
if actual_sha != sha256:
raise HTTPException(
400,
f"SHA-256 mismatch: expected {sha256}, got {actual_sha}",
)
meta = STORE.store(
sha256=sha256,
data=data,
content_type=file.content_type or "application/octet-stream",
source=source or file.filename or "",
build_id=build_id,
target=target,
runtime=runtime,
node=node,
)
return {"status": "stored", "sha256": sha256, "meta": meta}
@router.delete("/{sha256}")
async def api_cas_delete(sha256: str):
"""Delete an artifact from the cache."""
if STORE.delete(sha256):
return {"status": "deleted", "sha256": sha256}
raise HTTPException(404, "artifact not found")
Reference in New Issue View Git Blame Copy Permalink