// Package cas provides a content-addressable store client for the // sorcery-go ↔ Fester shared artifact cache. // // After BundleSovereign produces a .svb file, the Cauldron can push it to // the shared CAS via PushArtifact. Fester's DAG executor checks the CAS // before dispatching builds — if the artifact already exists (any runtime, // any node), the build is skipped entirely. // // The CAS is keyed by SHA-256 of the content. This means: // - An artifact built inside LXC on Node A is instantly available to // a Firecracker microVM on Node B. // - A .svb bundle produced by sorcery-go on the master node is // immediately available to all Fester workers. // - The same library compiled with the same toolchain on different // runtimes will deduplicate if the output matches. // // API contract (mirrors Fester's /api/cas/ endpoints): // // PUT /api/cas/{sha256} — store an artifact // GET /api/cas/{sha256} — retrieve an artifact (streamed) // HEAD /api/cas/{sha256} — check existence // DELETE /api/cas/{sha256} — remove an artifact // GET /api/cas/ — list all artifacts // GET /api/cas/stats — cache statistics // // Usage: // // client := cas.NewClient(festerURL) // // // After BundleSovereign: // sha, err := client.PushFile(ctx, "/path/to/output.svb", cas.ArtifactMeta{ // Source: "output.svb", // Target: "x86_64-linux-gnu", // Runtime: "podman", // }) // // // Before dispatching a build: // hit, err := client.CheckArtifact(ctx, actionHash) // if hit != nil { // // Skip the build — artifact already cached // } package cas import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "time" ) // ArtifactMeta is the metadata attached to a CAS entry. type ArtifactMeta struct { Source string `json:"source"` // original filename (e.g., "busybox-x86_64.svb") BuildID string `json:"build_id,omitempty"` // sorcery-go or Fester build ID Target string `json:"target,omitempty"` // build target (e.g., "x86_64-linux-gnu") Runtime string `json:"runtime,omitempty"` // execution runtime (e.g., "podman", "firecracker") Node string `json:"node,omitempty"` // node name that produced the artifact } // CASEntry is the metadata returned by HEAD /api/cas/{sha256}. type CASEntry struct { SHA256 string `json:"sha256"` Size int64 `json:"size"` ContentType string `json:"content_type"` Source string `json:"source"` BuildID string `json:"build_id"` Target string `json:"target"` Runtime string `json:"runtime"` Node string `json:"node"` CreatedAt float64 `json:"created_at"` LastAccessed float64 `json:"last_accessed"` AccessCount int `json:"access_count"` } // CASStats is returned by GET /api/cas/stats. type CASStats struct { TotalArtifacts int `json:"total_artifacts"` TotalBytes int64 `json:"total_bytes"` MaxBytes int64 `json:"max_bytes"` UtilizationPct float64 `json:"utilization_pct"` Hits int `json:"hits"` Misses int `json:"misses"` Stores int `json:"stores"` HitRatePct float64 `json:"hit_rate_pct"` } // Client is a content-addressable store client that talks to Fester's // /api/cas/ endpoints. type Client struct { baseURL string http *http.Client userAgent string } // NewClient creates a CAS client pointing at a Fester instance's CAS API. // The base URL should be the Fester root (e.g., "http://fester-master:8080"). // The stack operates behind a firewall (OPNsense/IPFire); no transport-layer // encryption is used. func NewClient(festerBaseURL string) *Client { return &Client{ baseURL: strings.TrimRight(festerBaseURL, "/"), http: &http.Client{ Timeout: 300 * time.Second, }, userAgent: "sorcery-go/cas (AGPL-3.0)", } } // CheckArtifact checks if an artifact exists in the shared CAS. // Returns the artifact metadata if found, nil if not cached. // // This is the key integration point: before sorcery-go dispatches a build // to Fester, it checks the CAS. If the artifact already exists, the build // can be skipped entirely — even if it was produced by a different node // or runtime. func (c *Client) CheckArtifact(ctx context.Context, sha256 string) (*CASEntry, error) { req, err := http.NewRequestWithContext(ctx, "HEAD", c.baseURL+"/api/cas/"+sha256, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", c.userAgent) resp, err := c.http.Do(req) if err != nil { return nil, fmt.Errorf("cas: check artifact: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return nil, nil // not cached — this is not an error } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return nil, fmt.Errorf("cas: HEAD %s → %d: %s", sha256, resp.StatusCode, string(body)) } var entry CASEntry if err := json.NewDecoder(resp.Body).Decode(&entry); err != nil { return nil, fmt.Errorf("cas: decode HEAD response: %w", err) } return &entry, nil } // PushArtifact stores a byte slice in the CAS. The sha256 parameter must // match the actual SHA-256 of data — the server verifies this. // Returns the SHA-256 on success. func (c *Client) PushArtifact(ctx context.Context, sha256 string, data []byte, meta ArtifactMeta) (string, error) { u := c.baseURL + "/api/cas/" + sha256 u += "?" + metaToQuery(meta).Encode() req, err := http.NewRequestWithContext(ctx, "PUT", u, bytes.NewReader(data)) if err != nil { return "", err } req.Header.Set("User-Agent", c.userAgent) req.Header.Set("Content-Type", "application/octet-stream") resp, err := c.http.Do(req) if err != nil { return "", fmt.Errorf("cas: push artifact: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) return "", fmt.Errorf("cas: PUT %s → %d: %s", sha256, resp.StatusCode, string(body)) } return sha256, nil } // MaxArtifactSize is the upper bound (2 GiB) for artifacts accepted by // PushFile and RetrieveArtifact. This prevents unbounded memory allocation // from a malicious or corrupted CAS server / oversized build output. const MaxArtifactSize int64 = 2 << 30 // 2 GiB // PushFile stores a file from disk in the CAS. It computes the SHA-256 // automatically. This is the primary method used after BundleSovereign // produces a .svb file. // // Returns the SHA-256 of the file on success. func (c *Client) PushFile(ctx context.Context, filePath string, meta ArtifactMeta) (string, error) { f, err := os.Open(filePath) if err != nil { return "", fmt.Errorf("cas: open %s: %w", filePath, err) } defer f.Close() // Reject files that exceed the artifact size limit before reading. info, err := f.Stat() if err != nil { return "", fmt.Errorf("cas: stat %s: %w", filePath, err) } if info.Size() > MaxArtifactSize { return "", fmt.Errorf("cas: %s (%d bytes) exceeds MaxArtifactSize (%d bytes)", filePath, info.Size(), MaxArtifactSize) } // Compute SHA-256 in a single pass hasher := sha256.New() var buf bytes.Buffer if _, err := io.Copy(io.MultiWriter(hasher, &buf), f); err != nil { return "", fmt.Errorf("cas: read %s: %w", filePath, err) } sum := hasher.Sum(nil) sha256Hex := hex.EncodeToString(sum) // Set source to the filename if not provided if meta.Source == "" { meta.Source = filepath.Base(filePath) } return c.PushArtifact(ctx, sha256Hex, buf.Bytes(), meta) } // RetrieveArtifact downloads an artifact from the CAS and returns its contents. // Returns nil if the artifact doesn't exist. func (c *Client) RetrieveArtifact(ctx context.Context, sha256 string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/cas/"+sha256, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", c.userAgent) resp, err := c.http.Do(req) if err != nil { return nil, fmt.Errorf("cas: retrieve artifact: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return nil, nil } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return nil, fmt.Errorf("cas: GET %s → %d: %s", sha256, resp.StatusCode, string(body)) } data, err := io.ReadAll(io.LimitReader(resp.Body, MaxArtifactSize)) if err != nil { return nil, fmt.Errorf("cas: read body: %w", err) } return data, nil } // RetrieveArtifactToFile downloads an artifact to a local file path. // Creates parent directories as needed. Returns the SHA-256 on success. func (c *Client) RetrieveArtifactToFile(ctx context.Context, sha256 string, destPath string) error { req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/cas/"+sha256, nil) if err != nil { return err } req.Header.Set("User-Agent", c.userAgent) resp, err := c.http.Do(req) if err != nil { return fmt.Errorf("cas: retrieve to file: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return fmt.Errorf("cas: artifact %s not found", sha256[:16]) } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return fmt.Errorf("cas: GET %s → %d: %s", sha256, resp.StatusCode, string(body)) } if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { return fmt.Errorf("cas: mkdir %s: %w", destPath, err) } out, err := os.Create(destPath) if err != nil { return fmt.Errorf("cas: create %s: %w", destPath, err) } defer out.Close() if _, err := io.Copy(out, resp.Body); err != nil { return fmt.Errorf("cas: write %s: %w", destPath, err) } return nil } // DeleteArtifact removes an artifact from the CAS. func (c *Client) DeleteArtifact(ctx context.Context, sha256 string) error { req, err := http.NewRequestWithContext(ctx, "DELETE", c.baseURL+"/api/cas/"+sha256, nil) if err != nil { return fmt.Errorf("cas: delete artifact: %w", err) } req.Header.Set("User-Agent", c.userAgent) resp, err := c.http.Do(req) if err != nil { return fmt.Errorf("cas: delete artifact: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return nil // already gone } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return fmt.Errorf("cas: DELETE %s → %d: %s", sha256, resp.StatusCode, string(body)) } return nil } // Stats returns the CAS cache statistics from Fester. func (c *Client) Stats(ctx context.Context) (*CASStats, error) { req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/cas/stats", nil) if err != nil { return nil, err } req.Header.Set("User-Agent", c.userAgent) req.Header.Set("Accept", "application/json") resp, err := c.http.Do(req) if err != nil { return nil, fmt.Errorf("cas: stats: %w", err) } defer resp.Body.Close() var stats CASStats if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil { return nil, fmt.Errorf("cas: decode stats: %w", err) } return &stats, nil } // FileSHA256 computes the SHA-256 hex digest of a file. // This is used to derive the CAS key before pushing or checking. func FileSHA256(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } // BytesSHA256 computes the SHA-256 hex digest of a byte slice. func BytesSHA256(data []byte) string { h := sha256.Sum256(data) return hex.EncodeToString(h[:]) } // metaToQuery converts ArtifactMeta to URL query parameters. func metaToQuery(m ArtifactMeta) url.Values { v := url.Values{} if m.Source != "" { v.Set("source", m.Source) } if m.BuildID != "" { v.Set("build_id", m.BuildID) } if m.Target != "" { v.Set("target", m.Target) } if m.Runtime != "" { v.Set("runtime", m.Runtime) } if m.Node != "" { v.Set("node", m.Node) } return v } // FormatBytes returns a human-readable size string (e.g., "1.5 GB"). func FormatBytes(bytes int64) string { const ( KB = 1024 MB = KB * 1024 GB = MB * 1024 ) switch { case bytes >= GB: return fmt.Sprintf("%.1f GB", float64(bytes)/float64(GB)) case bytes >= MB: return fmt.Sprintf("%.1f MB", float64(bytes)/float64(MB)) case bytes >= KB: return fmt.Sprintf("%.1f KB", float64(bytes)/float64(KB)) default: return strconv.FormatInt(bytes, 10) + " B" } }