292 lines
12 KiB
Go
Executable File
292 lines
12 KiB
Go
Executable File
// Package cast implements the multi-stage Cast pipeline.
|
|
//
|
|
// A Cast is no longer a single script execution. It is a pipeline:
|
|
//
|
|
// 1. Resolve Sub-Depends — feature-aware DAG solver
|
|
// 2. Generate Variant Hash — sha256(version + flags + arch + toolchain)
|
|
// 3. Cache Hit Check — skip rebuild if variant already in Tomb
|
|
// 4. Summon — download source tarball + verify hash
|
|
// 5. Unpack — extract tarball into sandbox source dir
|
|
// 6. ICE — run CONFIGURE, persist y/n answers
|
|
// 7. Sandbox Build — OverlayFS + namespaces, stream logs
|
|
// 8. Warding Inspect — verify Merkle root of produced files
|
|
// 9. Commit to Tomb — atomic blob ingest + epitaph write
|
|
// 10. Journal Update — mark StateInstalled
|
|
//
|
|
// Every phase publishes events to the EventBus under the cast's taskID so
|
|
// the CLI, TUI, and WebUI all see the same real-time progress.
|
|
package cast
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"dcos.net/sorcery-go/pkg/config"
|
|
"dcos.net/sorcery-go/pkg/dag"
|
|
"dcos.net/sorcery-go/pkg/eventbus"
|
|
"dcos.net/sorcery-go/pkg/grimoire"
|
|
"dcos.net/sorcery-go/pkg/sandbox"
|
|
"dcos.net/sorcery-go/pkg/state"
|
|
"dcos.net/sorcery-go/pkg/tomb"
|
|
)
|
|
|
|
// Pipeline is one cast operation.
|
|
type Pipeline struct {
|
|
Cfg *config.Config
|
|
Spell *grimoire.Spell
|
|
TargetArch string
|
|
Options map[string]bool // y/n answers from the Tablet / ICE
|
|
Toolchain string
|
|
Linkage string // "dynamic" or "static"
|
|
State *state.Manager
|
|
Tomb *tomb.Tomb
|
|
Graph *dag.Graph
|
|
Bus *eventbus.Bus
|
|
TaskID string
|
|
DryRun bool
|
|
Reconfigure bool
|
|
}
|
|
|
|
// Execute runs the full pipeline. Returns the EssenceID on success.
|
|
func (p *Pipeline) Execute(ctx context.Context) (string, error) {
|
|
p.Bus.Phase(p.TaskID, "resolving")
|
|
p.Bus.Log(p.TaskID, fmt.Sprintf("🔮 Casting %s %s (target=%s, linkage=%s)",
|
|
p.Spell.Name, p.Spell.Version, p.TargetArch, p.Linkage))
|
|
|
|
// Phase 1: Resolve sub-depends.
|
|
solver := &dag.Solver{Lookup: p.featureLookup}
|
|
reforges, err := solver.Solve(p.Spell.Name, p.Graph)
|
|
if err != nil {
|
|
p.Bus.Failed(p.TaskID, "sub-depends: "+err.Error())
|
|
return "", fmt.Errorf("cast: sub-depends: %w", err)
|
|
}
|
|
if len(reforges) > 0 {
|
|
p.Bus.Failed(p.TaskID, fmt.Sprintf("re-forge required: %+v", reforges))
|
|
return "", fmt.Errorf("cast: re-forge required: %+v", reforges)
|
|
}
|
|
|
|
// Phase 2: Variant hash.
|
|
variant := p.VariantHash()
|
|
p.Bus.Log(p.TaskID, "Variant hash: "+variant[:16]+"...")
|
|
|
|
// Phase 3: Cache hit.
|
|
if existing, err := p.Tomb.FindByVariant(variant); err == nil && existing != "" {
|
|
p.Bus.Log(p.TaskID, "✓ Cache hit — Essence already in Tomb: "+existing)
|
|
p.Bus.Complete(p.TaskID, existing)
|
|
return existing, nil
|
|
}
|
|
|
|
// Journal: mark StateSummoning.
|
|
if err := p.State.RecordJournal(state.JournalEntry{
|
|
SpellName: p.Spell.Name, Variant: variant,
|
|
Status: state.StateSummoning, TaskID: p.TaskID,
|
|
}); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// Phase 4: Summon (download + verify hash).
|
|
p.Bus.Phase(p.TaskID, "summoning")
|
|
if len(p.Spell.SourceURLs) == 0 {
|
|
p.Bus.Failed(p.TaskID, "SPELL has no SOURCE_URL defined")
|
|
return "", fmt.Errorf("cast: no SOURCE_URLs in DETAILS for %s", p.Spell.Name)
|
|
}
|
|
sourceURL := p.Spell.SourceURLs[0]
|
|
if sourceURL == "" {
|
|
p.Bus.Failed(p.TaskID, "no SOURCE_URL in DETAILS")
|
|
return "", fmt.Errorf("cast: no SOURCE_URL in DETAILS for %s", p.Spell.Name)
|
|
}
|
|
tarballPath := filepath.Join(p.Cfg.SpoolDir,
|
|
fmt.Sprintf("%s-%s.tar", p.Spell.Name, p.Spell.Version))
|
|
if err := Summon(ctx, sourceURL, tarballPath, p.Spell.SourceHash, p.Bus, p.TaskID); err != nil {
|
|
p.Bus.Failed(p.TaskID, "summon: "+err.Error())
|
|
return "", fmt.Errorf("cast: summon: %w", err)
|
|
}
|
|
|
|
// Phase 5: Sandbox setup + Unpack.
|
|
p.Bus.Phase(p.TaskID, "unpacking")
|
|
box := sandbox.New(p.Cfg.BuildRoot, p.Spell.Name+"-"+p.TaskID)
|
|
if err := box.Mount(); err != nil {
|
|
p.Bus.Log(p.TaskID, "OverlayFS unavailable, falling back to plain dir: "+err.Error())
|
|
box.SetFallback()
|
|
}
|
|
defer box.Cleanup()
|
|
|
|
if err := p.State.RecordJournal(state.JournalEntry{
|
|
SpellName: p.Spell.Name, Variant: variant,
|
|
Status: state.StateUnpacking, TaskID: p.TaskID,
|
|
}); err != nil {
|
|
return "", err
|
|
}
|
|
srcDir := filepath.Join(box.MountDir, "usr/src", fmt.Sprintf("%s-%s", p.Spell.Name, p.Spell.Version))
|
|
if err := Unpack(tarballPath, srcDir, p.Bus, p.TaskID); err != nil {
|
|
p.Bus.Failed(p.TaskID, "unpack: "+err.Error())
|
|
return "", fmt.Errorf("cast: unpack: %w", err)
|
|
}
|
|
box.Env["SOURCE_DIRECTORY"] = srcDir
|
|
|
|
// Apply linkage flags.
|
|
if err := applyLinkage(box, p.Linkage); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// Phase 6: ICE — run CONFIGURE if present.
|
|
p.Bus.Phase(p.TaskID, "configuring")
|
|
if p.Options == nil {
|
|
p.Options = make(map[string]bool)
|
|
}
|
|
if configurePath := filepath.Join(p.Spell.Directory, "CONFIGURE"); fileExists(configurePath) {
|
|
collected, err := RunICE(p.State, p.Spell, configurePath, p.Reconfigure, p.Bus, p.TaskID)
|
|
if err != nil {
|
|
p.Bus.Log(p.TaskID, "ICE: "+err.Error())
|
|
}
|
|
for k, v := range collected {
|
|
p.Options[k] = v
|
|
}
|
|
}
|
|
|
|
// Phase 7: Run BUILD.
|
|
p.Bus.Phase(p.TaskID, "building")
|
|
if err := p.State.RecordJournal(state.JournalEntry{
|
|
SpellName: p.Spell.Name, Variant: variant,
|
|
Status: state.StateCasting, TaskID: p.TaskID,
|
|
}); err != nil {
|
|
return "", err
|
|
}
|
|
buildPath := filepath.Join(p.Spell.Directory, "BUILD")
|
|
if !fileExists(buildPath) {
|
|
p.Bus.Failed(p.TaskID, "no BUILD script in "+p.Spell.Directory)
|
|
return "", fmt.Errorf("cast: no BUILD script in %s", p.Spell.Directory)
|
|
}
|
|
if err := box.Run(ctx, buildPath, p.Bus, p.TaskID); err != nil {
|
|
_ = p.State.RecordJournal(state.JournalEntry{
|
|
SpellName: p.Spell.Name, Variant: variant,
|
|
Status: state.StateFailed, TaskID: p.TaskID,
|
|
})
|
|
p.Bus.Failed(p.TaskID, "build failed: "+err.Error())
|
|
return "", fmt.Errorf("cast: build script failed: %w", err)
|
|
}
|
|
|
|
// Phase 8: Collect manifest + ingest blobs into the Tomb.
|
|
p.Bus.Phase(p.TaskID, "committing")
|
|
files, err := box.CollectManifest()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(files) == 0 {
|
|
p.Bus.Log(p.TaskID, "warning: BUILD produced no files — creating empty Essence")
|
|
}
|
|
fileHashes := make(map[string]string, len(files))
|
|
for i, f := range files {
|
|
hash, err := p.Tomb.IngestBlob(f)
|
|
if err != nil {
|
|
p.Bus.Failed(p.TaskID, fmt.Sprintf("ingest %s: %v", f, err))
|
|
return "", fmt.Errorf("cast: ingest %s: %w", f, err)
|
|
}
|
|
// Path relative to the sandbox upper dir.
|
|
rel := strings.TrimPrefix(f, box.WorkDir)
|
|
if rel == f {
|
|
// fallback mode — strip the MountDir prefix
|
|
rel = strings.TrimPrefix(f, box.MountDir)
|
|
}
|
|
fileHashes[rel] = hash
|
|
p.Bus.Progress(p.TaskID, i+1, len(files))
|
|
}
|
|
|
|
// Phase 9: Build Sarcophagus + store in Tomb.
|
|
sarc := &tomb.Sarcophagus{
|
|
SpellName: p.Spell.Name,
|
|
Version: p.Spell.Version,
|
|
VariantHash: variant,
|
|
Arch: p.TargetArch,
|
|
Linkage: p.Linkage,
|
|
Config: p.Options,
|
|
Files: fileHashes,
|
|
Toolchain: p.Toolchain,
|
|
License: p.Spell.License,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := p.Tomb.Store(sarc); err != nil {
|
|
p.Bus.Failed(p.TaskID, "tomb store: "+err.Error())
|
|
return "", fmt.Errorf("cast: tomb store: %w", err)
|
|
}
|
|
|
|
// Phase 10: Journal update.
|
|
if err := p.State.RecordJournal(state.JournalEntry{
|
|
SpellName: p.Spell.Name, Variant: variant,
|
|
Status: state.StateInstalled, TaskID: p.TaskID,
|
|
}); err != nil {
|
|
p.Bus.Log(p.TaskID, "warning: journal update failed: "+err.Error())
|
|
}
|
|
p.Bus.Complete(p.TaskID, sarc.EssenceID)
|
|
p.Bus.Log(p.TaskID, "✓ Essence sealed: "+sarc.EssenceID)
|
|
return sarc.EssenceID, nil
|
|
}
|
|
|
|
// VariantHash is the "Soul" of a binary — every unique combination of
|
|
// (version, y/n flags, arch, toolchain, linkage) produces a unique hash.
|
|
func (p *Pipeline) VariantHash() string {
|
|
keys := make([]string, 0, len(p.Options))
|
|
for k := range p.Options {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
parts := []string{p.Spell.Name, p.Spell.Version}
|
|
for _, k := range keys {
|
|
if p.Options[k] {
|
|
parts = append(parts, k+"=1")
|
|
} else {
|
|
parts = append(parts, k+"=0")
|
|
}
|
|
}
|
|
parts = append(parts, p.TargetArch, p.Toolchain, p.Linkage)
|
|
h := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
|
|
// featureLookup is the callback the dag.Solver uses to ask whether the
|
|
// currently-active Essence variant exposes a feature. It queries the Tomb.
|
|
func (p *Pipeline) featureLookup(spell, feature string) (bool, error) {
|
|
all, err := p.Tomb.List()
|
|
if err != nil {
|
|
return false, fmt.Errorf("cast: feature lookup: %w", err)
|
|
}
|
|
for _, s := range all {
|
|
if s.SpellName == spell && s.Config[feature] {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
// linkageFlags maps LinkStrategy to LDFLAGS and CC overrides.
|
|
var linkageFlags = map[string][2]string{
|
|
"static": {" -static -static-libgcc -static-libstdc++", "musl-gcc"},
|
|
"dynamic": {" -Wl,-rpath,/lib:/usr/lib", ""},
|
|
"hermetic": {" -static", "musl-gcc"},
|
|
}
|
|
|
|
func applyLinkage(box *sandbox.Box, linkage string) error {
|
|
if flags, ok := linkageFlags[linkage]; ok {
|
|
if box.Env == nil {
|
|
box.Env = make(map[string]string)
|
|
}
|
|
if flags[1] != "" {
|
|
box.Env["CC"] = flags[1]
|
|
}
|
|
box.Env["LDFLAGS"] = box.Env["LDFLAGS"] + flags[0]
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func fileExists(path string) bool {
|
|
_, err := os.Stat(path)
|
|
return err == nil
|
|
}
|