410 lines
14 KiB
Go
Executable File
410 lines
14 KiB
Go
Executable File
// Package cas provides a content-addressable store client for the
|
|
// sorcery-go ↔ Fester shared artifact cache.
|
|
//
|
|
// After BundleSovereign produces a .svb file, the Cauldron can push it to
|
|
// the shared CAS via PushArtifact. Fester's DAG executor checks the CAS
|
|
// before dispatching builds — if the artifact already exists (any runtime,
|
|
// any node), the build is skipped entirely.
|
|
//
|
|
// The CAS is keyed by SHA-256 of the content. This means:
|
|
// - An artifact built inside LXC on Node A is instantly available to
|
|
// a Firecracker microVM on Node B.
|
|
// - A .svb bundle produced by sorcery-go on the master node is
|
|
// immediately available to all Fester workers.
|
|
// - The same library compiled with the same toolchain on different
|
|
// runtimes will deduplicate if the output matches.
|
|
//
|
|
// API contract (mirrors Fester's /api/cas/ endpoints):
|
|
//
|
|
// PUT /api/cas/{sha256} — store an artifact
|
|
// GET /api/cas/{sha256} — retrieve an artifact (streamed)
|
|
// HEAD /api/cas/{sha256} — check existence
|
|
// DELETE /api/cas/{sha256} — remove an artifact
|
|
// GET /api/cas/ — list all artifacts
|
|
// GET /api/cas/stats — cache statistics
|
|
//
|
|
// Usage:
|
|
//
|
|
// client := cas.NewClient(festerURL)
|
|
//
|
|
// // After BundleSovereign:
|
|
// sha, err := client.PushFile(ctx, "/path/to/output.svb", cas.ArtifactMeta{
|
|
// Source: "output.svb",
|
|
// Target: "x86_64-linux-gnu",
|
|
// Runtime: "podman",
|
|
// })
|
|
//
|
|
// // Before dispatching a build:
|
|
// hit, err := client.CheckArtifact(ctx, actionHash)
|
|
// if hit != nil {
|
|
// // Skip the build — artifact already cached
|
|
// }
|
|
package cas
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ArtifactMeta is the metadata attached to a CAS entry.
|
|
type ArtifactMeta struct {
|
|
Source string `json:"source"` // original filename (e.g., "busybox-x86_64.svb")
|
|
BuildID string `json:"build_id,omitempty"` // sorcery-go or Fester build ID
|
|
Target string `json:"target,omitempty"` // build target (e.g., "x86_64-linux-gnu")
|
|
Runtime string `json:"runtime,omitempty"` // execution runtime (e.g., "podman", "firecracker")
|
|
Node string `json:"node,omitempty"` // node name that produced the artifact
|
|
}
|
|
|
|
// CASEntry is the metadata returned by HEAD /api/cas/{sha256}.
|
|
type CASEntry struct {
|
|
SHA256 string `json:"sha256"`
|
|
Size int64 `json:"size"`
|
|
ContentType string `json:"content_type"`
|
|
Source string `json:"source"`
|
|
BuildID string `json:"build_id"`
|
|
Target string `json:"target"`
|
|
Runtime string `json:"runtime"`
|
|
Node string `json:"node"`
|
|
CreatedAt float64 `json:"created_at"`
|
|
LastAccessed float64 `json:"last_accessed"`
|
|
AccessCount int `json:"access_count"`
|
|
}
|
|
|
|
// CASStats is returned by GET /api/cas/stats.
|
|
type CASStats struct {
|
|
TotalArtifacts int `json:"total_artifacts"`
|
|
TotalBytes int64 `json:"total_bytes"`
|
|
MaxBytes int64 `json:"max_bytes"`
|
|
UtilizationPct float64 `json:"utilization_pct"`
|
|
Hits int `json:"hits"`
|
|
Misses int `json:"misses"`
|
|
Stores int `json:"stores"`
|
|
HitRatePct float64 `json:"hit_rate_pct"`
|
|
}
|
|
|
|
// Client is a content-addressable store client that talks to Fester's
|
|
// /api/cas/ endpoints.
|
|
type Client struct {
|
|
baseURL string
|
|
http *http.Client
|
|
userAgent string
|
|
}
|
|
|
|
// NewClient creates a CAS client pointing at a Fester instance's CAS API.
|
|
// The base URL should be the Fester root (e.g., "http://fester-master:8080").
|
|
// The stack operates behind a firewall (OPNsense/IPFire); no transport-layer
|
|
// encryption is used.
|
|
func NewClient(festerBaseURL string) *Client {
|
|
return &Client{
|
|
baseURL: strings.TrimRight(festerBaseURL, "/"),
|
|
http: &http.Client{
|
|
Timeout: 300 * time.Second,
|
|
},
|
|
userAgent: "sorcery-go/cas (AGPL-3.0)",
|
|
}
|
|
}
|
|
|
|
// CheckArtifact checks if an artifact exists in the shared CAS.
|
|
// Returns the artifact metadata if found, nil if not cached.
|
|
//
|
|
// This is the key integration point: before sorcery-go dispatches a build
|
|
// to Fester, it checks the CAS. If the artifact already exists, the build
|
|
// can be skipped entirely — even if it was produced by a different node
|
|
// or runtime.
|
|
func (c *Client) CheckArtifact(ctx context.Context, sha256 string) (*CASEntry, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "HEAD", c.baseURL+"/api/cas/"+sha256, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("User-Agent", c.userAgent)
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cas: check artifact: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return nil, nil // not cached — this is not an error
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
return nil, fmt.Errorf("cas: HEAD %s → %d: %s", sha256, resp.StatusCode, string(body))
|
|
}
|
|
|
|
var entry CASEntry
|
|
if err := json.NewDecoder(resp.Body).Decode(&entry); err != nil {
|
|
return nil, fmt.Errorf("cas: decode HEAD response: %w", err)
|
|
}
|
|
|
|
return &entry, nil
|
|
}
|
|
|
|
// PushArtifact stores a byte slice in the CAS. The sha256 parameter must
|
|
// match the actual SHA-256 of data — the server verifies this.
|
|
// Returns the SHA-256 on success.
|
|
func (c *Client) PushArtifact(ctx context.Context, sha256 string, data []byte, meta ArtifactMeta) (string, error) {
|
|
u := c.baseURL + "/api/cas/" + sha256
|
|
u += "?" + metaToQuery(meta).Encode()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "PUT", u, bytes.NewReader(data))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("User-Agent", c.userAgent)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("cas: push artifact: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
|
return "", fmt.Errorf("cas: PUT %s → %d: %s", sha256, resp.StatusCode, string(body))
|
|
}
|
|
|
|
return sha256, nil
|
|
}
|
|
|
|
// MaxArtifactSize is the upper bound (2 GiB) for artifacts accepted by
|
|
// PushFile and RetrieveArtifact. This prevents unbounded memory allocation
|
|
// from a malicious or corrupted CAS server / oversized build output.
|
|
const MaxArtifactSize int64 = 2 << 30 // 2 GiB
|
|
|
|
// PushFile stores a file from disk in the CAS. It computes the SHA-256
|
|
// automatically. This is the primary method used after BundleSovereign
|
|
// produces a .svb file.
|
|
//
|
|
// Returns the SHA-256 of the file on success.
|
|
func (c *Client) PushFile(ctx context.Context, filePath string, meta ArtifactMeta) (string, error) {
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("cas: open %s: %w", filePath, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
// Reject files that exceed the artifact size limit before reading.
|
|
info, err := f.Stat()
|
|
if err != nil {
|
|
return "", fmt.Errorf("cas: stat %s: %w", filePath, err)
|
|
}
|
|
if info.Size() > MaxArtifactSize {
|
|
return "", fmt.Errorf("cas: %s (%d bytes) exceeds MaxArtifactSize (%d bytes)",
|
|
filePath, info.Size(), MaxArtifactSize)
|
|
}
|
|
|
|
// Compute SHA-256 in a single pass
|
|
hasher := sha256.New()
|
|
var buf bytes.Buffer
|
|
if _, err := io.Copy(io.MultiWriter(hasher, &buf), f); err != nil {
|
|
return "", fmt.Errorf("cas: read %s: %w", filePath, err)
|
|
}
|
|
|
|
sum := hasher.Sum(nil)
|
|
sha256Hex := hex.EncodeToString(sum)
|
|
|
|
// Set source to the filename if not provided
|
|
if meta.Source == "" {
|
|
meta.Source = filepath.Base(filePath)
|
|
}
|
|
|
|
return c.PushArtifact(ctx, sha256Hex, buf.Bytes(), meta)
|
|
}
|
|
|
|
// RetrieveArtifact downloads an artifact from the CAS and returns its contents.
|
|
// Returns nil if the artifact doesn't exist.
|
|
func (c *Client) RetrieveArtifact(ctx context.Context, sha256 string) ([]byte, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/cas/"+sha256, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("User-Agent", c.userAgent)
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cas: retrieve artifact: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return nil, nil
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
return nil, fmt.Errorf("cas: GET %s → %d: %s", sha256, resp.StatusCode, string(body))
|
|
}
|
|
|
|
data, err := io.ReadAll(io.LimitReader(resp.Body, MaxArtifactSize))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cas: read body: %w", err)
|
|
}
|
|
|
|
return data, nil
|
|
}
|
|
|
|
// RetrieveArtifactToFile downloads an artifact to a local file path.
|
|
// Creates parent directories as needed. Returns the SHA-256 on success.
|
|
func (c *Client) RetrieveArtifactToFile(ctx context.Context, sha256 string, destPath string) error {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/cas/"+sha256, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("User-Agent", c.userAgent)
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("cas: retrieve to file: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return fmt.Errorf("cas: artifact %s not found", sha256[:16])
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
return fmt.Errorf("cas: GET %s → %d: %s", sha256, resp.StatusCode, string(body))
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
|
|
return fmt.Errorf("cas: mkdir %s: %w", destPath, err)
|
|
}
|
|
|
|
out, err := os.Create(destPath)
|
|
if err != nil {
|
|
return fmt.Errorf("cas: create %s: %w", destPath, err)
|
|
}
|
|
defer out.Close()
|
|
|
|
if _, err := io.Copy(out, resp.Body); err != nil {
|
|
return fmt.Errorf("cas: write %s: %w", destPath, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// DeleteArtifact removes an artifact from the CAS.
|
|
func (c *Client) DeleteArtifact(ctx context.Context, sha256 string) error {
|
|
req, err := http.NewRequestWithContext(ctx, "DELETE", c.baseURL+"/api/cas/"+sha256, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("cas: delete artifact: %w", err)
|
|
}
|
|
req.Header.Set("User-Agent", c.userAgent)
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("cas: delete artifact: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return nil // already gone
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
return fmt.Errorf("cas: DELETE %s → %d: %s", sha256, resp.StatusCode, string(body))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Stats returns the CAS cache statistics from Fester.
|
|
func (c *Client) Stats(ctx context.Context) (*CASStats, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/cas/stats", nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("User-Agent", c.userAgent)
|
|
req.Header.Set("Accept", "application/json")
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cas: stats: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
var stats CASStats
|
|
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
|
return nil, fmt.Errorf("cas: decode stats: %w", err)
|
|
}
|
|
return &stats, nil
|
|
}
|
|
|
|
// FileSHA256 computes the SHA-256 hex digest of a file.
|
|
// This is used to derive the CAS key before pushing or checking.
|
|
func FileSHA256(path string) (string, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer f.Close()
|
|
|
|
h := sha256.New()
|
|
if _, err := io.Copy(h, f); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
|
|
// BytesSHA256 computes the SHA-256 hex digest of a byte slice.
|
|
func BytesSHA256(data []byte) string {
|
|
h := sha256.Sum256(data)
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
|
|
// metaToQuery converts ArtifactMeta to URL query parameters.
|
|
func metaToQuery(m ArtifactMeta) url.Values {
|
|
v := url.Values{}
|
|
if m.Source != "" {
|
|
v.Set("source", m.Source)
|
|
}
|
|
if m.BuildID != "" {
|
|
v.Set("build_id", m.BuildID)
|
|
}
|
|
if m.Target != "" {
|
|
v.Set("target", m.Target)
|
|
}
|
|
if m.Runtime != "" {
|
|
v.Set("runtime", m.Runtime)
|
|
}
|
|
if m.Node != "" {
|
|
v.Set("node", m.Node)
|
|
}
|
|
return v
|
|
}
|
|
|
|
// FormatBytes returns a human-readable size string (e.g., "1.5 GB").
|
|
func FormatBytes(bytes int64) string {
|
|
const (
|
|
KB = 1024
|
|
MB = KB * 1024
|
|
GB = MB * 1024
|
|
)
|
|
switch {
|
|
case bytes >= GB:
|
|
return fmt.Sprintf("%.1f GB", float64(bytes)/float64(GB))
|
|
case bytes >= MB:
|
|
return fmt.Sprintf("%.1f MB", float64(bytes)/float64(MB))
|
|
case bytes >= KB:
|
|
return fmt.Sprintf("%.1f KB", float64(bytes)/float64(KB))
|
|
default:
|
|
return strconv.FormatInt(bytes, 10) + " B"
|
|
}
|
|
} |