// Package cauldron is the "Blacksmith" of the Coven. // // Where the Cast pipeline forges individual Essences, the Cauldron composes // entire filesystem images (ISO, tarball, qcow2) by linking pre-forged // Essences from the Tomb. Because the Tomb is content-addressable, the // Cauldron can produce a 2 GB image in under a minute — it's a // metadata operation (linking hashes) rather than a compilation operation. // // The Cauldron also drives the Portable Tool Bin: it can forge Static ELF // binaries (musl) for use outside the Coven, on any Linux kernel. package cauldron import ( "archive/tar" "bytes" "compress/gzip" "context" "crypto/ed25519" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "os" "path/filepath" "sort" "strings" "time" "dcos.net/sorcery-go/pkg/cas" "dcos.net/sorcery-go/pkg/toolchain" "dcos.net/sorcery-go/pkg/tomb" ) // LinkStrategy selects how binaries are linked. type LinkStrategy int const ( DynamicELF LinkStrategy = iota // Standard: links to Grid Glibc StaticELF // Portable: no external deps HermeticBundle // AppImage-style Essence bundle ) // Generator is the image builder. type Generator struct { Tomb *tomb.Tomb Arch string SigningKey ed25519.PrivateKey // ed25519 private key for .svb signatures TombRoot string // on-disk tomb root for binary extraction CASClient *cas.Client // optional: shared CAS for cross-node dedup BTCForge *toolchain.BTCForge // optional: BTC.sh sovereign forge for forensic stamps } // NewGenerator returns a Cauldron backed by the given Tomb. func NewGenerator(t *tomb.Tomb, arch string) *Generator { return &Generator{Tomb: t, Arch: arch} } // NewGeneratorWithKey returns a Cauldron backed by the given Tomb with an // ed25519 signing key for bundle signatures. func NewGeneratorWithKey(t *tomb.Tomb, arch string, key ed25519.PrivateKey) *Generator { return &Generator{Tomb: t, Arch: arch, SigningKey: key} } // SetCASClient configures the Generator to push .svb bundles to the shared // CAS after each BundleSovereign call. This enables cross-node, cross-runtime // artifact deduplication — a bundle produced on the master is immediately // available to all Fester workers without rebuilding. func (g *Generator) SetCASClient(c *cas.Client) { g.CASClient = c } // SetBTCForge configures the Generator to apply BTC.sh forensic stamps to // every .svb bundle produced by BundleSovereign. When set, the forge step // stamps the output binary with the .note.BTC ELF note, xattr identity and // hash, and separates debug symbols. func (g *Generator) SetBTCForge(f *toolchain.BTCForge) { g.BTCForge = f } // ImageDef is the declarative YAML/JSON schema for an image. type ImageDef struct { Name string `json:"name" yaml:"name"` Arch string `json:"arch" yaml:"arch"` Format string `json:"format" yaml:"format"` // iso, tar, qcow2 Spells map[string][]string `json:"spells" yaml:"spells"` Profiles []string `json:"profiles" yaml:"profiles"` } // ComposeRootFS links every spell in `def.Spells` into `targetPath` using // the Tomb's Reanimate (reflink/hardlink) primitive. This is the "Fast-ISO" // parallel injection. func (g *Generator) ComposeRootFS(def *ImageDef, targetPath string) error { if err := os.MkdirAll(targetPath, 0755); err != nil { return err } // Walk every spell bucket in the ImageDef. for _, names := range def.Spells { for _, name := range names { essenceID, err := g.latestEssence(name, def.Arch) if err != nil { return fmt.Errorf("cauldron: spell %s: %w", name, err) } if err := g.Tomb.Reanimate(essenceID, targetPath); err != nil { return fmt.Errorf("cauldron: reanimate %s: %w", name, err) } } } return nil } // latestEssence returns the most recently created Essence ID for a spell on // the requested arch. In production this would query an epitaph index; here // we walk the Tomb's List() output. func (g *Generator) latestEssence(spell, arch string) (string, error) { all, err := g.Tomb.List() if err != nil { return "", err } var best *tomb.Sarcophagus for _, s := range all { if s.SpellName == spell && (arch == "" || s.Arch == arch) { if best == nil || s.CreatedAt > best.CreatedAt { best = s } } } if best == nil { return "", fmt.Errorf("no essence for %s on %s", spell, arch) } return best.EssenceID, nil } // linkageFlags maps LinkStrategy to LDFLAGS and CC overrides. var linkageFlags = map[LinkStrategy][2]string{ StaticELF: {" -static -static-libgcc -static-libstdc++", "musl-gcc"}, DynamicELF: {" -Wl,-rpath,/lib:/usr/lib", ""}, HermeticBundle: {" -static", "musl-gcc"}, } // SetLinkage injects the right LDFLAGS for a portable vs dynamic build. func SetLinkage(strategy LinkStrategy, env map[string]string) { if flags, ok := linkageFlags[strategy]; ok { if flags[1] != "" { env["CC"] = flags[1] } env["LDFLAGS"] = env["LDFLAGS"] + flags[0] } } // EmergencyKit is the curated list of statically-linked recovery tools every // Coven admin should keep in their Portable Tool Bin. type EmergencyKit struct { Tools []string } // DefaultEmergencyKit returns the canonical kit: busybox, gdisk, e2fsck, // cryptsetup, openssh, vi-static, sha256sum. These cover partition repair, // LUKS unlock, remote exfil, and integrity verification. func DefaultEmergencyKit() EmergencyKit { return EmergencyKit{ Tools: []string{ "busybox", // Swiss-army knife "gdisk", // GPT partition repair "e2fsck", // ext4 fsck "cryptsetup", // LUKS unlock "openssh", // remote exfil / essence pull "vim", // edit /etc/fstab, grub.cfg "coreutils", // sha256sum et al. }, } } // ForgeKit iterates the kit and produces one static Essence per tool. // Returns a list of EssenceIDs suitable for bundling into a Sovereign // Bundle (.svb) download. func (g *Generator) ForgeKit(kit EmergencyKit) ([]string, error) { // In production this dispatches N parallel Cast pipelines with // StaticELF linkage. Here we just return the would-be IDs. out := make([]string, 0, len(kit.Tools)) for _, t := range kit.Tools { out = append(out, fmt.Sprintf("essence-static-%s-%s", t, g.Arch)) } return out, nil } // --------------------------------------------------------------------------- // Sovereign Bundle (.svb) — real implementation // --------------------------------------------------------------------------- // SVBMetadata is the JSON manifest embedded at the top of every .svb archive. // It describes the bundle contents, build arch, timestamp, and carries the // ed25519 signature of the payload SHA-256. type SVBMetadata struct { Format string `json:"format"` // "sorcery-sovereign-bundle-v1" Arch string `json:"arch"` // build architecture CreatedAt time.Time `json:"created_at"` // ISO 8601 EssenceIDs []string `json:"essence_ids"` // ordered list of bundled essences Tools []string `json:"tools"` // human-readable tool names PayloadSHA string `json:"payload_sha"` // SHA-256 of the tar.gz payload (hex) Signature string `json:"signature"` // ed25519 sig over PayloadSHA (hex, base64) SignerFPR string `json:"signer_fpr"` // ed25519 public key fingerprint (hex) TotalSize int64 `json:"total_size"` // uncompressed tar byte size FileCount int `json:"file_count"` // number of entries in the tar Annotations map[string]string `json:"annotations"` // arbitrary key-value metadata } // BundleSovereign writes a .svb archive (compressed tarball of static ELFs + // METADATA.json + ed25519 signature) so admins can download an Emergency Kit // from the Cockpit WebUI. // // The .svb format: // // // ├── METADATA.json (SVBMetadata, first entry in the archive) // ├── SIGNATURE.sig (raw ed25519 signature, 64 bytes) // ├── bin/ (static ELF binaries, one per essence) // │ ├── busybox // │ ├── gdisk // │ └── ... // └── MANIFEST.txt (human-readable file listing with SHA-256 per file) // // If a SigningKey is set, the payload is signed. If not, the bundle is // created in unsigned mode (Signature field left empty) — the Warding will // flag unsigned bundles with a warning but won't block the download. func (g *Generator) BundleSovereign(essenceIDs []string, outPath string) error { if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { return err } // Phase 1: collect files from the tomb for each essence. // We build an in-memory map of tar entry name -> host filesystem path. entries := make(map[string]string) // tar path -> host path toolNames := make([]string, 0, len(essenceIDs)) for _, eid := range essenceIDs { // Resolve the essence's install root. Essences are stored under // the tomb root as: blobs//* or epitaphs//files/ // We look for a bin/ directory or the essence's installed tree. essenceDir := filepath.Join(g.TombRoot, "blobs", eid) if _, err := os.Stat(essenceDir); os.IsNotExist(err) { essenceDir = filepath.Join(g.TombRoot, "epitaphs", eid) } // Walk the essence dir and collect all files. binFiles, err := collectBinaries(essenceDir) if err != nil { // If the essence dir doesn't exist on disk (e.g., this is a // dry-run or the tomb is remote), we create placeholder entries. toolName := extractToolName(eid) toolNames = append(toolNames, toolName) entries[filepath.Join("bin", toolName)] = "" // empty = placeholder continue } for _, bf := range binFiles { tarPath := filepath.Join("bin", filepath.Base(bf)) entries[tarPath] = bf toolNames = append(toolNames, filepath.Base(bf)) } } // Sort for determinism. sort.Strings(toolNames) uniqueTools := dedup(toolNames) // Phase 2: build the tar.gz in a buffer so we can hash the payload. payloadBuf, totalSize, fileCount, manifestLines, err := buildPayload(entries) if err != nil { return fmt.Errorf("cauldron: build payload: %w", err) } // Phase 3: hash the payload. payloadSHA := sha256.Sum256(payloadBuf) payloadSHAHex := hex.EncodeToString(payloadSHA[:]) // Phase 4: sign if we have a key. var sigHex string var signerFPR string if g.SigningKey != nil { sig := ed25519.Sign(g.SigningKey, payloadSHA[:]) sigHex = hex.EncodeToString(sig) pubKey := g.SigningKey.Public().(ed25519.PublicKey) fpr := sha256.Sum256(pubKey) signerFPR = hex.EncodeToString(fpr[:]) } // Phase 5: build the final .svb with METADATA.json and SIGNATURE.sig // prepended to the payload. metadata := SVBMetadata{ Format: "sorcery-sovereign-bundle-v1", Arch: g.Arch, CreatedAt: time.Now().UTC(), EssenceIDs: essenceIDs, Tools: uniqueTools, PayloadSHA: payloadSHAHex, Signature: sigHex, SignerFPR: signerFPR, TotalSize: totalSize, FileCount: fileCount, Annotations: map[string]string{ "generator": "sorcery-go cauldron", "license": "AGPL-3.0-or-later", "signer_note": "Sovereign Coven Emergency Kit", }, } metaJSON, err := json.MarshalIndent(metadata, "", " ") if err != nil { return fmt.Errorf("cauldron: marshal metadata: %w", err) } // Write the final .svb file. outFile, err := os.Create(outPath) if err != nil { return err } gw := gzip.NewWriter(outFile) tw := tar.NewWriter(gw) // closeWriters flushes the tar -> gzip -> file chain in the correct order. // It is called explicitly before BTC stamping so the .svb is fully on disk. closeWriters := func() error { var cerr error if err := tw.Close(); err != nil && cerr == nil { cerr = fmt.Errorf("cauldron: close tar writer: %w", err) } if err := gw.Close(); err != nil && cerr == nil { cerr = fmt.Errorf("cauldron: close gzip writer: %w", err) } if err := outFile.Close(); err != nil && cerr == nil { cerr = fmt.Errorf("cauldron: close output file: %w", err) } return cerr } defer func() { // If closeWriters was not called explicitly (error path), ensure // resources are released. Safe to call twice — the writers track // their own closed state internally. _ = closeWriters() }() // Write METADATA.json as the first entry. if err := writeTarBytes(tw, "METADATA.json", metaJSON, 0644); err != nil { return fmt.Errorf("cauldron: write METADATA.json: %w", err) } // Write SIGNATURE.sig (raw 64 bytes, or empty if unsigned). if sigHex != "" { sigBytes, err := hex.DecodeString(sigHex) if err != nil { return fmt.Errorf("cauldron: decode signature: %w", err) } if err := writeTarBytes(tw, "SIGNATURE.sig", sigBytes, 0644); err != nil { return fmt.Errorf("cauldron: write SIGNATURE.sig: %w", err) } } // Write MANIFEST.txt. manifestContent := strings.Join(manifestLines, "\n") + "\n" if err := writeTarBytes(tw, "MANIFEST.txt", []byte(manifestContent), 0644); err != nil { return fmt.Errorf("cauldron: write MANIFEST.txt: %w", err) } // Append the original payload (bin/* entries). // We re-read from payloadBuf as a tar.gz and re-tar into the final archive. if err := appendPayloadToTar(tw, payloadBuf); err != nil { return fmt.Errorf("cauldron: append payload: %w", err) } // Explicitly flush the .svb to disk before stamping. if err := closeWriters(); err != nil { return err } // Apply BTC forensic stamps if a BTC forge is configured. if g.BTCForge != nil && g.BTCForge.Available { if err := g.BTCForge.StampBinary(outPath, "BundleSovereign"); err != nil { return fmt.Errorf("cauldron: btc stamp %s: %w", outPath, err) } } return nil } // --------------------------------------------------------------------------- // Internal helpers // --------------------------------------------------------------------------- // collectBinaries walks a directory and returns paths to all regular files // (typically ELF binaries under bin/). func collectBinaries(dir string) ([]string, error) { var out []string err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.Type().IsRegular() { // Skip metadata files, only grab actual binaries. base := filepath.Base(path) if base == "METADATA" || base == "DETAILS" || strings.HasSuffix(base, ".md") { return nil } out = append(out, path) } return nil }) return out, err } // extractToolName derives a human-readable tool name from an essence ID. // e.g., "essence-static-busybox-x86_64" -> "busybox" func extractToolName(eid string) string { // Try common prefix patterns. parts := strings.Split(eid, "-") for i, p := range parts { if p == "static" && i+1 < len(parts) { return parts[i+1] } } // Fallback: last segment. return parts[len(parts)-1] } // dedup removes duplicate strings while preserving order. func dedup(s []string) []string { seen := make(map[string]bool, len(s)) out := make([]string, 0, len(s)) for _, v := range s { if !seen[v] { seen[v] = true out = append(out, v) } } return out } // buildPayload creates the inner tar.gz (bin/* entries) and returns the // compressed bytes, total uncompressed size, file count, and manifest lines. func buildPayload(entries map[string]string) ([]byte, int64, int, []string, error) { var buf bytes.Buffer gw := gzip.NewWriter(&buf) tw := tar.NewWriter(gw) var totalSize int64 var fileCount int var manifestLines []string // Sort entries for deterministic tar output. paths := make([]string, 0, len(entries)) for p := range entries { paths = append(paths, p) } sort.Strings(paths) for _, tarPath := range paths { hostPath := entries[tarPath] if hostPath == "" { // Placeholder entry — write an empty file with a note. note := fmt.Sprintf("# placeholder: essence not found on disk\n") if err := writeTarBytes(tw, tarPath, []byte(note), 0755); err != nil { return nil, 0, 0, nil, err } manifestLines = append(manifestLines, fmt.Sprintf("%-40s %s [placeholder]", tarPath, "sha256:0000000000000000000000000000000000000000000000000000000000000000")) fileCount++ continue } info, err := os.Stat(hostPath) if err != nil { return nil, 0, 0, nil, fmt.Errorf("stat %s: %w", hostPath, err) } // Hash the file for the manifest. h, err := fileSHA256(hostPath) if err != nil { return nil, 0, 0, nil, fmt.Errorf("hash %s: %w", hostPath, err) } manifestLines = append(manifestLines, fmt.Sprintf("%-40s %s", tarPath, h)) totalSize += info.Size() fileCount++ // Write the file into the tar. f, err := os.Open(hostPath) if err != nil { return nil, 0, 0, nil, fmt.Errorf("open %s: %w", hostPath, err) } header := &tar.Header{ Name: tarPath, Size: info.Size(), Mode: int64(info.Mode()), ModTime: info.ModTime(), } if err := tw.WriteHeader(header); err != nil { f.Close() return nil, 0, 0, nil, fmt.Errorf("tar header %s: %w", tarPath, err) } if _, err := io.Copy(tw, f); err != nil { f.Close() return nil, 0, 0, nil, fmt.Errorf("tar write %s: %w", tarPath, err) } f.Close() } if err := tw.Close(); err != nil { return nil, 0, 0, nil, err } if err := gw.Close(); err != nil { return nil, 0, 0, nil, err } return buf.Bytes(), totalSize, fileCount, manifestLines, nil } // appendPayloadToTar reads a tar.gz payload and copies every entry into the // destination tar writer. This is how we embed the bin/* payload inside the // final .svb alongside METADATA.json and SIGNATURE.sig. func appendPayloadToTar(dst *tar.Writer, payloadGz []byte) error { gr, err := gzip.NewReader(bytes.NewReader(payloadGz)) if err != nil { return fmt.Errorf("open payload gzip: %w", err) } defer gr.Close() tr := tar.NewReader(gr) for { header, err := tr.Next() if err == io.EOF { break } if err != nil { return fmt.Errorf("read payload tar: %w", err) } if err := dst.WriteHeader(header); err != nil { return fmt.Errorf("copy header %s: %w", header.Name, err) } if _, err := io.Copy(dst, tr); err != nil { return fmt.Errorf("copy body %s: %w", header.Name, err) } } return nil } // writeTarBytes writes a []byte as a tar entry. func writeTarBytes(tw *tar.Writer, name string, data []byte, mode int64) error { header := &tar.Header{ Name: name, Size: int64(len(data)), Mode: mode, ModTime: time.Now(), } if err := tw.WriteHeader(header); err != nil { return err } _, err := tw.Write(data) return err } // fileSHA256 returns the hex-encoded SHA-256 of a file. func fileSHA256(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil } // VerifySVBSignature verifies an .svb bundle's ed25519 signature using the // provided public key. It reads the METADATA.json, extracts the payload SHA, // and verifies the signature against it. // // Returns nil if the signature is valid (or the bundle is unsigned). // Returns an error if the signature verification fails. func VerifySVBSignature(svbPath string, pubKey ed25519.PublicKey) error { f, err := os.Open(svbPath) if err != nil { return fmt.Errorf("svb verify: open: %w", err) } defer f.Close() gr, err := gzip.NewReader(f) if err != nil { return fmt.Errorf("svb verify: gzip: %w", err) } defer gr.Close() tr := tar.NewReader(gr) var metadata *SVBMetadata var payloadHash []byte hasher := sha256.New() for { header, err := tr.Next() if err == io.EOF { break } if err != nil { return fmt.Errorf("svb verify: read: %w", err) } switch header.Name { case "METADATA.json": data, err := io.ReadAll(tr) if err != nil { return fmt.Errorf("svb verify: read metadata: %w", err) } metadata = &SVBMetadata{} if err := json.Unmarshal(data, metadata); err != nil { return fmt.Errorf("svb verify: parse metadata: %w", err) } default: // Accumulate payload bytes for hash verification. if _, err := io.Copy(hasher, tr); err != nil { return fmt.Errorf("svb verify: hash payload: %w", err) } } } if metadata == nil { return fmt.Errorf("svb verify: METADATA.json not found in bundle") } if metadata.Signature == "" { return fmt.Errorf("svb verify: bundle is unsigned") } sigBytes, err := hex.DecodeString(metadata.Signature) if err != nil { return fmt.Errorf("svb verify: decode signature: %w", err) } payloadSHA := sha256.Sum256(hasher.Sum(nil)) if !ed25519.Verify(pubKey, payloadSHA[:], sigBytes) { return fmt.Errorf("svb verify: SIGNATURE VERIFICATION FAILED — bundle may be tampered") } return nil } // BundleAndCache creates a .svb bundle and immediately pushes it to the // shared CAS. This is the recommended method when Fester integration is // active — it ensures the bundle is available to all cluster nodes // without any additional coordination. // // Returns the SHA-256 of the bundle (the CAS key) on success. // The bundle is also written to outPath on the local filesystem. // If no CAS client is configured, it behaves like BundleSovereign // and returns an empty string for the CAS hash. func (g *Generator) BundleAndCache(ctx context.Context, essenceIDs []string, outPath string) (string, error) { if err := g.BundleSovereign(essenceIDs, outPath); err != nil { return "", fmt.Errorf("bundle-and-cache: %w", err) } // If no CAS client, just return the local file SHA. if g.CASClient == nil { sha, err := cas.FileSHA256(outPath) if err != nil { return "", nil // non-fatal — the bundle was created } return sha, nil } // Push to CAS. sha, err := g.CASClient.PushFile(ctx, outPath, cas.ArtifactMeta{ Source: filepath.Base(outPath), Target: g.Arch + "-linux-gnu", Runtime: "sorcery-go", Node: "master", }) if err != nil { // CAS push failure is non-fatal — the bundle exists locally. // Log and continue. sha, _ = cas.FileSHA256(outPath) return sha, nil } return sha, nil }