// Package toolchain provides BTC.sh (Build Tool Chain) integration for // sorcery-go. BTC.sh forges a sovereign, forensically-stamped GCC toolchain // whose provenance can be verified through ELF notes and extended filesystem // attributes. // // BTC 0.4.0+ supports multi-architecture cross-compilation: // // x86_64: haswell, haswell-ep, skylake, skylake-x, skylake-server, // znver1, znver2, znver3, znver4, // apu-zn1, apu-zn2, apu-zn3, apu-zn4, // atom-silvermont, atom-goldmont, atom-tremont, atom-sierraforest // mipsel: mipselr2 (MIPS32R2 LE, o32 ABI, musl) // arm: armv7 (Cortex-A NEON hard-float, musl) // tilegx: tilegx (Tilera TILE-Gx72, musl) // // This file implements probing, environment setup, stamp verification, and // binary stamping that mirrors BTC.sh's own f_stamp_binary and verification // routines. package toolchain import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "log" "os" "os/exec" "path/filepath" "regexp" "runtime" "sort" "strings" "dcos.net/sorcery-go/pkg/config" ) // BTCStamp holds the forensic identification data extracted from a BTC-built // binary. BTC.sh stamps every compiled artifact with two mechanisms: // // 1. An ELF NOTE section named ".note.BTC" containing human-readable fields. // 2. Extended attributes (xattr): user.btc.identity and user.btc.hash. type BTCStamp struct { Org string // Origin organization (e.g., "dcos.net") Kernel string // Kernel version at build time Arch string // Target architecture / target ID Label string // SYS_LABEL — unique forge identifier Forge string // Build step name (e.g., "stage2-gcc") Identity string // xattr user.btc.identity value Hash string // xattr user.btc.hash (SHA-256 of the binary) HasNote bool // .note.BTC section present HasXAttr bool // xattr identity present Valid bool // true if Hash matches the binary's actual SHA-256 } // BTCForge represents a probed BTC.sh installation with its golden image. // BTC 0.4.0+ populates CrossMode, TargetID, TargetTriple, and CLib from // the manifest JSON sidecar written by BTC.sh. type BTCForge struct { Config *config.Config SYSLabel string // auto-detected or from config GoldenImage string // path to the {SYS_LABEL}-toolchain-golden.tar.xz Available bool // true if BTC.sh and golden image are usable TargetArch string // base architecture (x86_64, arm, mipsel, tilegx) ExtractedDir string // path where the golden image is extracted (if any) // BTC 0.4.0 multi-arch fields (populated from manifest JSON) CrossMode bool // true if this is a cross-compiled toolchain TargetID string // BTC target identifier (haswell, znver3, armv7, etc.) TargetTriple string // GCC target triple (e.g., arm-dcosnet-linux-musleabihf) TargetMarch string // -march value (e.g., armv7-a, mips32r2, tilegx) CLib string // C library: "glibc" or "musl" Family string // architecture family (intel, amd, mips, arm, tile) Manifest *BTCManifest } // BTCManifest is the JSON structure of the {SYS_LABEL}-manifest.json // sidecar written by BTC.sh 0.4.0+. It contains structured metadata // that is more reliable than parsing filenames. type BTCManifest struct { BTCVersion string `json:"btc_version"` Mode string `json:"mode"` CrossMode bool `json:"cross_mode"` SysLabel string `json:"sys_label"` TargetID string `json:"target_id"` TargetArch string `json:"target_arch"` TargetCPU string `json:"target_cpu"` TargetMarch string `json:"target_march"` TargetTriple string `json:"target_triple"` HostArch string `json:"host_arch"` ISATag string `json:"isa_tag"` OptTag string `json:"opt_tag"` ABI string `json:"abi"` CLib string `json:"clib"` Endian string `json:"endian"` Family string `json:"family"` Description string `json:"description"` KernelMin string `json:"kernel_min"` Kernel string `json:"kernel"` Binutils string `json:"binutils"` GCC string `json:"gcc"` GLibc string `json:"glibc"` Musl string `json:"musl"` Libxcrypt string `json:"libxcrypt"` GoldenImage string `json:"golden_image"` CFlags string `json:"cflags"` Ldflags string `json:"ldflags"` } // ISA flag table — maps ISA tag (uppercase) to compiler flags. // Table-driven lookup replaces if/else chains (SEI CERT CTR50-JP). var isaFlags = map[string]string{ "AVX2": " -mavx2", "AVX512": " -mavx512f -mavx512dq -mavx512vl -mavx512bw", "SSE4_2": " -msse4.2", "NEON": " -mfpu=neon -mfloat-abi=hard", "MIPS32": "", "TILE": "", } // Probe checks whether BTC.sh and its golden image exist on disk and returns // a BTCForge struct describing what was found. If the config supplies a // BTCSYSLabel it is used directly; otherwise the label is parsed from the // golden image filename. // // For BTC 0.4.0+, the manifest JSON sidecar is loaded to populate // CrossMode, TargetID, TargetTriple, CLib, and other structured fields. func Probe(cfg *config.Config) *BTCForge { forge := &BTCForge{ Config: cfg, } // Check BTC.sh exists and is executable. info, err := os.Stat(cfg.BTCPath) if err != nil || info.IsDir() { return forge } if info.Mode()&0111 == 0 { return forge } // Check BTCRoot for a golden image matching *-toolchain-golden.tar.xz. pattern := filepath.Join(cfg.BTCRoot, "*-toolchain-golden.tar.xz") matches, err := filepath.Glob(pattern) if err != nil || len(matches) == 0 { return forge } // Use the first match. If multiple golden images exist the caller can // disambiguate via BTCSYSLabel. goldenPath := matches[0] base := filepath.Base(goldenPath) sysLabel := parseGoldenLabel(base) // If the config provides a label, verify it matches or use the config // value and search for the corresponding image. if cfg.BTCSYSLabel != "" { candidate := filepath.Join(cfg.BTCRoot, cfg.BTCSYSLabel+"-toolchain-golden.tar.xz") if _, err := os.Stat(candidate); err == nil { sysLabel = cfg.BTCSYSLabel goldenPath = candidate } else { // Config label does not match any file; use the discovered one. sysLabel = cfg.BTCSYSLabel } } forge.SYSLabel = sysLabel forge.GoldenImage = goldenPath forge.Available = true forge.TargetArch = cfg.HostArch if forge.TargetArch == "" { forge.TargetArch = hostArch() } // Try to load the manifest for richer metadata (BTC 0.4.0+). if manifest := loadManifest(cfg.BTCRoot, sysLabel); manifest != nil { forge.Manifest = manifest forge.CrossMode = manifest.CrossMode forge.TargetID = manifest.TargetID forge.TargetTriple = manifest.TargetTriple forge.TargetMarch = manifest.TargetMarch forge.CLib = manifest.CLib forge.Family = manifest.Family forge.TargetArch = manifest.TargetArch } return forge } // loadManifest reads a BTC manifest JSON sidecar if it exists. func loadManifest(btcRoot, sysLabel string) *BTCManifest { manifestPath := filepath.Join(btcRoot, sysLabel+"-manifest.json") data, err := os.ReadFile(manifestPath) if err != nil { return nil } var m BTCManifest if err := json.Unmarshal(data, &m); err != nil { log.Printf("btc: warning: failed to parse manifest %s: %v", manifestPath, err) return nil } return &m } // ListTargets scans the BTC root for all available golden images and // returns a slice of BTCForge structs, one per found toolchain. This // is useful for the WebUI "Toolchain Lab" view to show operators which // cross-compilers are available. func ListTargets(btcRoot string) []*BTCForge { pattern := filepath.Join(btcRoot, "*-toolchain-golden.tar.xz") matches, err := filepath.Glob(pattern) if err != nil { return nil } sort.Strings(matches) forges := make([]*BTCForge, 0, len(matches)) for _, goldenPath := range matches { base := filepath.Base(goldenPath) sysLabel := parseGoldenLabel(base) if sysLabel == "" { continue } forge := &BTCForge{ SYSLabel: sysLabel, GoldenImage: goldenPath, Available: true, } // Load manifest for structured metadata. if manifest := loadManifest(btcRoot, sysLabel); manifest != nil { forge.Manifest = manifest forge.CrossMode = manifest.CrossMode forge.TargetID = manifest.TargetID forge.TargetTriple = manifest.TargetTriple forge.TargetMarch = manifest.TargetMarch forge.CLib = manifest.CLib forge.Family = manifest.Family forge.TargetArch = manifest.TargetArch } else { // Legacy 0.3.x: parse from SYS_LABEL. forge.TargetArch = hostArch() forge.CLib = "glibc" } forges = append(forges, forge) } return forges } // parseGoldenLabel extracts the SYS_LABEL from a golden image filename of // the form "{SYS_LABEL}-toolchain-golden.tar.xz". func parseGoldenLabel(filename string) string { base := strings.TrimSuffix(filename, "-toolchain-golden.tar.xz") if base == filename { return "" } return base } // SetExtractedDir records where the golden image has been extracted so that // BuildEnv can point PATH at the correct bin/ subdirectories. func (f *BTCForge) SetExtractedDir(dir string) { f.ExtractedDir = dir } // BuildEnv returns the environment variables needed for a Cast pipeline to // compile with the BTC sovereign toolchain. The returned map is intended to // be merged into the sandbox environment. // // For BTC 0.4.0+, if a manifest is loaded, the CFLAGS and triple come from // the manifest's structured data rather than being guessed from the label. // Cross-compiled toolchains use {triple}-gcc/{triple}-g++ naming. // // If the golden image has been extracted (ExtractedDir is set), PATH is // pointed at the extracted tree. Otherwise, BTC's GLOBAL_CFLAGS are // assembled with --sysroot pointing at BTCRoot. func (f *BTCForge) BuildEnv() map[string]string { env := make(map[string]string) if !f.Available { return env } // Determine the sysroot and bin directory. newroot := f.BTCRoot binDir := "" if f.ExtractedDir != "" { newroot = f.ExtractedDir binDir = filepath.Join(f.ExtractedDir, "bin") if _, err := os.Stat(binDir); err != nil { binDir = filepath.Join(f.ExtractedDir, "usr", "bin") } } // Determine the target triple and march. // Prefer manifest data (BTC 0.4.0+), fall back to derivation. triple := f.TargetTriple march := f.TargetMarch if triple == "" { // Legacy 0.3.x derivation arch := f.TargetArch if arch == "" { arch = hostArch() } triple = arch + "-dcosnet-linux-gnu" } if march == "" { march = f.TargetID if march == "" { march = f.TargetArch } } // PATH: prepend the BTC bin directories. if binDir != "" { env["PATH"] = binDir + ":" + os.Getenv("PATH") } // Compiler executables. // Cross-toolchains (0.4.0+) use {triple}-gcc naming. // Native toolchains may have plain gcc in the sysroot. if f.CrossMode && triple != "" { env["CC"] = triple + "-gcc" env["CXX"] = triple + "-g++" } else { env["CC"] = triple + "-gcc" env["CXX"] = triple + "-g++" } // Build CFLAGS. // For BTC 0.4.0+ with a manifest, use the manifest's cflags. // Otherwise, assemble from march + ISA + sysroot. var cflags string if f.Manifest != nil && f.Manifest.CFlags != "" { // Manifest cflags may contain --sysroot pointing at the build-time // cleanroom. Rewrite to use the actual sysroot/newroot. cflags = rewriteSysroot(f.Manifest.CFlags, newroot) } else { // Legacy assembly. cflags = fmt.Sprintf("-O3 -march=%s -flto -ffat-lto-objects --sysroot=%s -pipe", march, newroot) } env["CFLAGS"] = cflags env["CXXFLAGS"] = cflags // BTC GLOBAL_LDFLAGS. var ldflags string if f.Manifest != nil && f.Manifest.Ldflags != "" { ldflags = rewriteSysroot(f.Manifest.Ldflags, newroot) } else { ldflags = fmt.Sprintf("-Wl,-O1 -Wl,--as-needed -flto --sysroot=%s", newroot) } env["LDFLAGS"] = ldflags // Forensic tracking. if f.SYSLabel != "" { env["BTC_SYS_LABEL"] = f.SYSLabel } env["BTC_MODE"] = "1" if f.TargetID != "" { env["BTC_TARGET_ID"] = f.TargetID } if f.CrossMode { env["BTC_CROSS"] = "1" } if f.CLib != "" { env["BTC_CLIB"] = f.CLib } if f.TargetTriple != "" { env["BTC_TARGET_TRIPLE"] = f.TargetTriple } return env } // rewriteSysroot replaces --sysroot= flags with --sysroot=. // This is needed when the manifest's cflags/ldflags contain the build-time // cleanroom path but the toolchain has been extracted to a different location. func rewriteSysroot(flags, newroot string) string { // Match --sysroot= (greedy to end of flag value). re := regexp.MustCompile(`--sysroot=\S+`) return re.ReplaceAllString(flags, "--sysroot="+newroot) } // VerifyStamp reads and verifies BTC forensic stamps from a binary. It uses // readelf to extract the .note.BTC ELF note section and getfattr to read the // extended filesystem attributes. func (f *BTCForge) VerifyStamp(binPath string) (*BTCStamp, error) { stamp := &BTCStamp{} // Verify the file exists. if _, err := os.Stat(binPath); err != nil { return nil, fmt.Errorf("btc: verify stamp: %w", err) } // Extract .note.BTC via readelf. noteOut, err := exec.Command("readelf", "-n", binPath).CombinedOutput() if err == nil { parsed := parseNoteBTC(string(noteOut)) if parsed != nil { stamp.Org = parsed.Org stamp.Kernel = parsed.Kernel stamp.Arch = parsed.Arch stamp.Label = parsed.Label stamp.Forge = parsed.Forge stamp.HasNote = parsed.HasNote } } // Extract xattrs via getfattr. xattrOut, err := exec.Command("getfattr", "-d", "--name=user.btc.identity", "--name=user.btc.hash", binPath).CombinedOutput() if err == nil { identity, hash := parseXAttrs(string(xattrOut)) stamp.Identity = identity stamp.Hash = hash stamp.HasXAttr = identity != "" } // Validate the hash against the actual file content. if stamp.Hash != "" { actual, err := fileSHA256(binPath) if err == nil { stamp.Valid = (actual == strings.TrimPrefix(stamp.Hash, "sha256:")) } } return stamp, nil } // StampBinary applies BTC forensic stamps to a compiled binary. It: // // 1. Creates a small assembly object containing the .note.BTC ELF note. // 2. Injects the note via objcopy --add-section. // 3. Sets extended attributes user.btc.identity and user.btc.hash. // 4. Separates debug symbols (objcopy --only-keep-debug, strip, add-gnu-debuglink). // // If objcopy or the assembler is unavailable the function logs a warning // and continues rather than failing the build. func (f *BTCForge) StampBinary(binPath string, forgeStep string) error { // Determine stamp fields. org := "dcos.net" kernel := detectKernel() arch := f.TargetID if arch == "" { arch = f.TargetArch } if arch == "" { arch = hostArch() } label := f.SYSLabel if label == "" { label = "unknown" } // Step 1: Create the .note.BTC assembly source in a temp directory. noteDir, err := os.MkdirTemp("", "btc-stamp-*") if err != nil { return fmt.Errorf("btc: stamp: create temp dir: %w", err) } defer os.RemoveAll(noteDir) asmSrc := buildNoteAsm(org, kernel, arch, label, forgeStep) asmPath := filepath.Join(noteDir, "btc_note.S") if err := os.WriteFile(asmPath, []byte(asmSrc), 0644); err != nil { return fmt.Errorf("btc: stamp: write asm: %w", err) } // Step 2: Assemble the note object. // For cross-compiled toolchains, use the cross-assembler when available. asmCmd := "as" if f.CrossMode && f.TargetTriple != "" { crossAsm := f.TargetTriple + "-as" if _, err := exec.LookPath(crossAsm); err == nil { asmCmd = crossAsm } } objPath := filepath.Join(noteDir, "btc_note.o") if err := exec.Command(asmCmd, "-o", objPath, asmPath).Run(); err != nil { log.Printf("btc: warning: assembler not available, skipping .note.BTC injection: %v", err) applyXAttrStamps(binPath, label) return nil } // Step 3: Copy the original binary with the note section injected. stampedPath := binPath + ".btc-stamped" if err := exec.Command("objcopy", "--add-section", ".note.BTC="+objPath, binPath, stampedPath).Run(); err != nil { log.Printf("btc: warning: objcopy not available, skipping .note.BTC injection: %v", err) applyXAttrStamps(binPath, label) return nil } // Replace the original with the stamped version. if err := os.Rename(stampedPath, binPath); err != nil { // Clean up the stamped copy on failure. os.Remove(stampedPath) return fmt.Errorf("btc: stamp: replace binary: %w", err) } // Step 4: Set extended attributes and separate debug symbols. applyXAttrStamps(binPath, label) return nil } // applyXAttrStamps sets the BTC identity and hash extended attributes // on a binary, then optionally separates debug symbols. func applyXAttrStamps(binPath, label string) { identity := label fileHash, err := fileSHA256(binPath) if err != nil { fileHash = "sha256:error" } setXAttr(binPath, "user.btc.identity", identity) setXAttr(binPath, "user.btc.hash", fileHash) separateDebugSymbols(binPath) } // --------------------------------------------------------------------------- // Internal helpers // --------------------------------------------------------------------------- // noteFieldRe matches the pipe-delimited fields inside a .note.BTC // readelf output block. BTC.sh writes a single line like: // // "Org: dcos.net|K:7.1|Arch:haswell|Label:DCOSNET-HASWELL-AVX2-LTO|Forge:binutils-configure" // // We parse the full pipe-delimited string to extract each field. var noteFieldRe = regexp.MustCompile(`Org:\s*([^|]+)\|K:([^|]+)\|Arch:([^|]+)\|Label:([^|]+)\|Forge:([^|]+)`) // parseNoteBTC extracts stamp fields from readelf -n output and returns // a BTCStamp with HasNote set to true. Returns nil if the .note.BTC // section is not found or contains no recognizable fields. func parseNoteBTC(readelfOutput string) *BTCStamp { s := &BTCStamp{} // Find the .note.BTC block. idx := strings.Index(readelfOutput, "note.BTC") if idx == -1 { return nil } // Parse only within the note.BTC section (stop at the next section). block := readelfOutput[idx:] if nextSection := strings.Index(block, "\nDisplaying notes found in:"); nextSection > 0 { block = block[:nextSection] } matches := noteFieldRe.FindStringSubmatch(block) if len(matches) >= 6 { s.Org = strings.TrimSpace(matches[1]) s.Kernel = strings.TrimSpace(matches[2]) s.Arch = strings.TrimSpace(matches[3]) s.Label = strings.TrimSpace(matches[4]) s.Forge = strings.TrimSpace(matches[5]) } // Only return if we found at least one field. if s.Org == "" && s.Kernel == "" && s.Arch == "" { return nil } s.HasNote = true return s } // parseXAttrs extracts user.btc.identity and user.btc.hash values from // getfattr -d output. The format is: // // user.btc.identity="BTC-SYS_LABEL-5.15.0-sovereign" // user.btc.hash="sha256:abcdef..." var xattrIdentityRe = regexp.MustCompile(`user\.btc\.identity="([^"]*)"`) var xattrHashRe = regexp.MustCompile(`user\.btc\.hash="([^"]*)"`) func parseXAttrs(getfattrOutput string) (identity, hash string) { if m := xattrIdentityRe.FindStringSubmatch(getfattrOutput); len(m) > 1 { identity = m[1] } if m := xattrHashRe.FindStringSubmatch(getfattrOutput); len(m) > 1 { hash = m[1] } return identity, hash } // buildNoteAsm generates an architecture-neutral ELF NOTE assembly source // that defines a .note.BTC section with the given fields. The output // uses raw .byte directives for portability across all ELF targets // (x86_64, arm, mipsel, tilegx). This mirrors BTC.sh's f_stamp_binary // routine which uses the same approach. func buildNoteAsm(org, kernel, arch, label, forge string) string { // Use the same pipe-delimited format as BTC.sh and Fester for // cross-project compatibility. BTC.sh's f_stamp_binary writes: // Org: dcos.net|K:7.1|Arch:haswell|Label:DCOSNET-HASWELL-AVX2-LTO|Forge:binutils-configure desc := fmt.Sprintf("Org: %s|K:%s|Arch:%s|Label:%s|Forge:%s", org, kernel, arch, label, forge) // Build the ELF NOTE in raw form using .byte directives. // ELF Note structure: namesz (4) + descsz (4) + type (4) + name + desc. name := "BTC" nameBytes := append([]byte(name), 0) descBytes := []byte(desc) noteType := uint32(1) // NT_VERSION — matches BTC.sh's .long 1 var b strings.Builder b.WriteString(" .section .note.BTC, \"a\", @note\n") b.WriteString(" .align 4\n") b.WriteString(" .globl __btc_note_start\n") b.WriteString("__btc_note_start:\n") // namesz (little-endian). b.WriteString(fmt.Sprintf(" .long %d\n", len(nameBytes))) // descsz. b.WriteString(fmt.Sprintf(" .long %d\n", len(descBytes))) // type. b.WriteString(fmt.Sprintf(" .long %d\n", noteType)) // name bytes. for _, c := range nameBytes { b.WriteString(fmt.Sprintf(" .byte %d\n", c)) } // Pad name to 4-byte alignment. if len(nameBytes)%4 != 0 { pad := 4 - (len(nameBytes) % 4) for i := 0; i < pad; i++ { b.WriteString(" .byte 0\n") } } // Description bytes. for _, c := range descBytes { b.WriteString(fmt.Sprintf(" .byte %d\n", c)) } // Pad description to 4-byte alignment. if len(descBytes)%4 != 0 { pad := 4 - (len(descBytes) % 4) for i := 0; i < pad; i++ { b.WriteString(" .byte 0\n") } } b.WriteString(" .align 4\n") b.WriteString(" .globl __btc_note_end\n") b.WriteString("__btc_note_end:\n") b.WriteString(" .previous\n") return b.String() } // detectKernel returns the running kernel version string (e.g., "5.15.0-generic"). func detectKernel() string { var utsname runtime.Utsname if err := runtime.Uname(&utsname); err != nil { return "unknown" } return strings.TrimRight(string(utsname.Release[:]), "\x00") } // fileSHA256 returns the hex-encoded SHA-256 of a file (without the "sha256:" prefix). func fileSHA256(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := h.ReadFrom(f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } // setXAttr sets an extended filesystem attribute on a file. If setfattr is // not available or the filesystem does not support xattrs, the function // logs a warning and continues. func setXAttr(path, attr, value string) { if err := exec.Command("setfattr", "-n", attr, "-v", value, path).Run(); err != nil { log.Printf("btc: warning: setfattr %s on %s failed: %v", attr, path, err) } } // separateDebugSymbols extracts debug info from a binary into a separate // .debug file, strips the binary, and links the debug file back. This // mirrors BTC.sh's debug symbol separation steps: // // objcopy --only-keep-debug {bin} {bin}.debug // strip --strip-unneeded {bin} // objcopy --add-gnu-debuglink={bin}.debug {bin} func separateDebugSymbols(binPath string) { debugPath := binPath + ".debug" // Extract debug symbols. if err := exec.Command("objcopy", "--only-keep-debug", binPath, debugPath).Run(); err != nil { log.Printf("btc: warning: objcopy --only-keep-debug failed for %s: %v", binPath, err) return } // Strip the binary. if err := exec.Command("strip", "--strip-unneeded", binPath).Run(); err != nil { log.Printf("btc: warning: strip failed for %s: %v", binPath, err) return } // Link the debug file back. if err := exec.Command("objcopy", "--add-gnu-debuglink="+debugPath, binPath).Run(); err != nil { log.Printf("btc: warning: objcopy --add-gnu-debuglink failed for %s: %v", binPath, err) } }