dcosnet
/
BuildToolChain
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug testing

This commit is contained in:
Jeremy Anderson 2026-08-08 23:58:47 -04:00
parent ea83eac47e
commit d2989812dd
4 changed files with 451 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

325
BTC.sh
View File

@ -3,23 +3,30 @@
# - [[ ... ]] conditionals, == pattern matching, =~ regex, (( )) arithmetic # - [[ ... ]] conditionals, == pattern matching, =~ regex, (( )) arithmetic
# - ${var^^} uppercase expansion, associative arrays, process substitution # - ${var^^} uppercase expansion, associative arrays, process substitution
# - set -euo pipefail for strict error handling # - set -euo pipefail for strict error handling
# BTC-0.4.0.sh - Build Tool Chain # BTC-${BTC_VERSION} - Build Tool Chain
# Identity: dcosnet / dcos.net | Multi-Arch Cross-Compilation Build Tool Chain # Identity: dcosnet / dcos.net | Multi-Arch Cross-Compilation Build Tool Chain
# Version: 0.4.1 | Persistence: /opt/BTC | Volatile: ramfs # Version: 0.4.2 | Persistence: /opt/BTC | Volatile: ramfs
# License: GNU AGPLv3 Mandatory Prominent Interactive Notice # License: GNU AGPLv3 Mandatory Prominent Interactive Notice
# Copyright (C) 2012-2026 Jeremy Anderson (info@dcos.net) # Copyright (C) 2012-2026 Jeremy Anderson (info@dcos.net)
set -euo pipefail set -euo pipefail
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
# ----------------------------------------------------------------------------
# Single source of truth for the BTC version. Every banner, header comment,
# manifest, and --help text MUST reference this constant — never hardcode the
# version string in multiple places.
# ----------------------------------------------------------------------------
readonly BTC_VERSION="0.4.2"
# ============================================================================ # ============================================================================
# 1. AGPL INTERACTIVE LICENSE COMPLIANCE # 1. AGPL INTERACTIVE LICENSE COMPLIANCE
# ============================================================================ # ============================================================================
function f_agpl_header() { function f_agpl_header() {
clear clear
cat << 'EOF' cat << EOF
=========================================================================== ===========================================================================
BTC-0.4.1.sh - Build Tool Chain (AGPLv3 PROTECTED) BTC-${BTC_VERSION} - Build Tool Chain (AGPLv3 PROTECTED)
Cross-Compilation Build Tool Chain Cross-Compilation Build Tool Chain
=========================================================================== ===========================================================================
This program is free software: you can redistribute it and/or modify it This program is free software: you can redistribute it and/or modify it
@ -177,7 +184,7 @@ BTC_TARGETS[tilegx]="tilegx|tilegx|tilegx|TILE|abi64|musl|little|tile|Tilera TIL
# --- Helper: list all registered targets --- # --- Helper: list all registered targets ---
function f_list_targets() { function f_list_targets() {
echo ">> BTC-0.4.0 Registered Cross-Compilation Targets:" echo ">> BTC-${BTC_VERSION} Registered Cross-Compilation Targets:"
echo ">> ===============================================" echo ">> ==============================================="
printf ">> %-16s %-10s %-18s %-8s %-6s %s\n" "TARGET_ID" "ARCH" "MARCH" "ISA" "CLIB" "DESCRIPTION" printf ">> %-16s %-10s %-18s %-8s %-6s %s\n" "TARGET_ID" "ARCH" "MARCH" "ISA" "CLIB" "DESCRIPTION"
printf ">> %-16s %-10s %-18s %-8s %-6s %s\n" "--------" "----" "-----" "---" "----" "-----------" printf ">> %-16s %-10s %-18s %-8s %-6s %s\n" "--------" "----" "-----" "---" "----" "-----------"
@ -337,9 +344,25 @@ function _configure_from_target() {
if [[ ${safe_threads} -gt ${total_cpus} ]]; then safe_threads=${total_cpus}; fi if [[ ${safe_threads} -gt ${total_cpus} ]]; then safe_threads=${total_cpus}; fi
export v_threads="-j${safe_threads}" export v_threads="-j${safe_threads}"
# Pin GCC version for Tile-Gx (upstream dropped after GCC 11) # Reset version overrides to the default baseline before per-target
# pinning. Without this reset, calling _configure_from_target twice
# in the same process (e.g. test harnesses, or a future batch-build
# mode) would leak the tile pin into subsequent non-tile targets.
# Single-target builds are unaffected, but the reset is cheap and
# makes the function idempotent.
v_gcc="${v_gcc_default}"
v_linux="${v_linux_default}"
v_linux_headers="${v_linux_headers_default}"
# Pin GCC and Linux versions for Tile-Gx.
# GCC upstream dropped mainline Tile-Gx support after GCC 11.
# Linux upstream removed the tile architecture in 5.9; the last LTS
# series that still builds for tilegx is 5.4.x. Both overrides MUST run
# before f_download(), which captures these globals into A_SRC_URL[].
if [[ "${BTC_T_FAMILY}" == "tile" ]]; then if [[ "${BTC_T_FAMILY}" == "tile" ]]; then
v_gcc='gcc-10.3.0' v_gcc="${v_gcc_tile}"
v_linux="${v_linux_tile}"
v_linux_headers="${v_linux_headers_tile}"
fi fi
echo ">> [IDENTITY STAMP] ${SYS_LABEL}" echo ">> [IDENTITY STAMP] ${SYS_LABEL}"
@ -361,9 +384,30 @@ export SOURCE_CACHE=${BTC_ARCHIVE}/src
export RAMDISK_SIZE="12gb" export RAMDISK_SIZE="12gb"
# Upstream Production Matrices (defaults — can be overridden per-target) # Upstream Production Matrices (defaults — can be overridden per-target)
v_linux='linux-7.1' # Versions track the current LFS stable baseline (LFS 13.0+) where practical,
# bumped forward to the latest point releases that are actually downloadable
# from upstream mirrors as of ${BTC_VERSION}.
#
# - linux-7.1.7 : latest stable point release on the 7.1.x series
# - binutils-2.46.1 : LFS-aligned binutils point release
# - gcc-15.3.0 : current GCC stable (LFS 13.0 ships 14.2.0; we bump
# to 15.3.0 to pick up znver4/sierraforest support
# and GCC 15 stricter const-correctness)
# - glibc-2.43 : current glibc stable
# - musl-1.2.6 : current musl stable
#
# Tile-Gx override: tile architecture was REMOVED from mainline Linux in 5.9
# (commit 65ad263b1d0f). The last LTS series with tile support is 5.4.x, so
# tile targets are pinned to linux-5.4.302 (LTS) and gcc-10.3.0 (last GCC
# release with full tile-gx backend). See _configure_from_target() for the
# runtime override.
v_linux='linux-7.1.7'
v_linux_default='linux-7.1.7'
v_linux_tile='linux-5.4.302'
v_binutils='binutils-2.46.1' v_binutils='binutils-2.46.1'
v_gcc='gcc-15.3.0' v_gcc='gcc-15.3.0'
v_gcc_default='gcc-15.3.0'
v_gcc_tile='gcc-10.3.0'
v_glibc='glibc-2.43' v_glibc='glibc-2.43'
v_libxcrypt='4.5.2' v_libxcrypt='4.5.2'
v_gmp='gmp-6.3.0' v_gmp='gmp-6.3.0'
@ -371,6 +415,8 @@ v_mpfr='mpfr-4.2.2'
v_mpc='mpc-1.4.0' v_mpc='mpc-1.4.0'
v_musl='musl-1.2.6' v_musl='musl-1.2.6'
v_linux_headers="${v_linux}" v_linux_headers="${v_linux}"
v_linux_headers_default="${v_linux_default}"
v_linux_headers_tile="${v_linux_tile}"
# These are set after f_resolve_target: # These are set after f_resolve_target:
# NEWROOT, LOGS, HOST_ARCH, TARGET, TARGET_ARCH, GLOBAL_CFLAGS, GLOBAL_LDFLAGS # NEWROOT, LOGS, HOST_ARCH, TARGET, TARGET_ARCH, GLOBAL_CFLAGS, GLOBAL_LDFLAGS
@ -434,22 +480,45 @@ function f_guard() {
} }
function f_entropy_shield() { function f_entropy_shield() {
local min_entropy=1000 # Modern Linux (>= 5.6) initializes the CRNG at boot via getrandom(2)
local cur_entropy # and the kernel's own jitter entropy collector. Since 5.6, the
if [[ -f /proc/sys/kernel/random/entropy_avail ]]; then # input-pool counter in /proc/sys/kernel/random/entropy_avail is
cur_entropy=$(< /proc/sys/kernel/random/entropy_avail) # capped at 256 by design — it does NOT reflect "available" entropy
if [[ ${cur_entropy} -lt ${min_entropy} ]]; then # in the 2.6-era sense anymore. Any value >= 256 means "CRNG ready,
echo ">> [ENTROPY DEFICIT] Pool dropped to ${cur_entropy}. Injecting safe hardware-jitter..." # getrandom will return immediately". The historical "1000" threshold
find /bin /sbin -type f -exec ls -l {} + > /dev/null 2>&1 & # is from the 2.6 era when /dev/random could genuinely block.
sleep 2 #
kill $! 2>/dev/null || true # Behavior on a modern box:
fi # - entropy_avail will be 256 essentially always (capped)
# - getrandom(GRND_NONBLOCK) succeeds instantly
# - The "ENTROPY DEFICIT" warning would fire on EVERY step and the
# old `sleep 2` blocked the parent pipeline for no reason.
#
# Fix: lower the threshold to 256, drop the blocking sleep entirely,
# fire the jitter injector in the background (fire-and-forget, never
# blocks the calling build step). If getrandom isn't usable on this
# host we still benefit from any extra jitter, but we don't pay any
# wall-clock time for it.
local min_entropy=256
local cur_entropy=0
if [[ -r /proc/sys/kernel/random/entropy_avail ]]; then
cur_entropy=$(< /proc/sys/kernel/random/entropy_avail) || cur_entropy=0
fi
if [[ ${cur_entropy} -gt 0 && ${cur_entropy} -lt ${min_entropy} ]]; then
echo ">> [ENTROPY] Pool at ${cur_entropy} (< ${min_entropy}); jitter injector fired in background."
# Fire-and-forget. Never blocks the calling build step.
# The injector walks /bin /sbin /usr/bin to feed disk-I/O timing
# jitter into the kernel input pool. If it fails or finds nothing
# useful, no harm done — getrandom(2) still works on modern kernels.
{ find /bin /sbin /usr/bin -type f -exec ls -l {} + > /dev/null 2>&1; } &
disown 2>/dev/null || true
fi fi
} }
function f_exec_log() { function f_exec_log() {
local cmd="$1" local cmd="$1"
local log_base="$2" local log_base="$2"
local log_file="${LOGS}/${log_base}.log"
f_entropy_shield f_entropy_shield
f_guard f_guard
@ -458,9 +527,25 @@ function f_exec_log() {
# Note: ${cmd} is sourced from internal build functions only (not user input). # Note: ${cmd} is sourced from internal build functions only (not user input).
# The trust boundary is the BTC.sh script itself — do not expose f_exec_log # The trust boundary is the BTC.sh script itself — do not expose f_exec_log
# as a public API with externally-supplied command strings. # as a public API with externally-supplied command strings.
stdbuf -oL -eL bash -c "${cmd}" 2>&1 | \ #
pv -t -r -b -N "${log_base}" | \ # Pipeline: bash -c "$cmd" → stdbuf (line-buffer) → pv (progress bar)
tee -a "${LOGS}/${log_base}.log" > /dev/null # → tee (log file) → /dev/null (suppress stdout so the terminal stays clean).
#
# On failure: the `if !` form suspends `set -e` for the condition test,
# captures the pipeline's exit status (with `set -o pipefail` this is
# the rightmost non-zero exit in the pipe — usually bash -c's exit code),
# then surfaces the last 40 lines of the log file to stderr so the
# operator sees the actual error inline instead of having to dig
# through ${LOGS}/<step>.log post-mortem.
if ! stdbuf -oL -eL bash -c "${cmd}" 2>&1 | \
pv -t -r -b -N "${log_base}" | \
tee -a "${log_file}" > /dev/null; then
echo ">> [FAILED] ${log_base} exited non-zero." >&2
echo ">> [FAILED] Last 40 lines of ${log_file}:" >&2
tail -n 40 "${log_file}" >&2 2>/dev/null || true
echo ">> [FAILED] Full log: ${log_file}" >&2
return 1
fi
} }
function f_tmux_dashboard() { function f_tmux_dashboard() {
@ -585,7 +670,8 @@ function f_stamp_binary() {
# The .note.BTC payload now carries the active sig tier and token, # The .note.BTC payload now carries the active sig tier and token,
# making each deployment's forensic identity distinct. # making each deployment's forensic identity distinct.
# Note type 0xB7C is a vendor-specific identifier — it does NOT # Note type 0xB7C is a vendor-specific identifier — it does NOT
# This vendor type (0xB7C) does not correspond to any standard. # correspond to any standard ELF note type (the Linux vendor-note
# namespace 0x0B7C is reserved for out-of-tree consumers).
cat << EOF > btc_stamp.s cat << EOF > btc_stamp.s
.section .note.BTC,"a",@note .section .note.BTC,"a",@note
.long 2f - 1f .long 2f - 1f
@ -629,18 +715,19 @@ EOF
# Associative array: package stem -> upstream URL # Associative array: package stem -> upstream URL
# Only packages consumed by the build sequence are listed. # Only packages consumed by the build sequence are listed.
# URLs are official upstream mirrors — replace to point at a local mirror. # URLs are official upstream mirrors — replace to point at a local mirror.
declare -A A_SRC_URL=( #
[binutils]="https://ftp.gnu.org/gnu/binutils/${v_binutils}.tar.xz" # NOTE: A_SRC_URL is declared here as an empty container and populated lazily
[linux]="https://cdn.kernel.org/pub/linux/kernel/v7.x/${v_linux}.tar.xz" # inside f_download(). This is required because v_linux, v_gcc, and
[linux-headers]="https://cdn.kernel.org/pub/linux/kernel/v7.x/${v_linux_headers}.tar.xz" # v_linux_headers may be overridden per-target (e.g. tile targets pin
[gcc]="https://ftp.gnu.org/gnu/gcc/${v_gcc}/${v_gcc}.tar.xz" # linux-5.4.302 and gcc-10.3.0 because mainline dropped tile support in
[glibc]="https://ftp.gnu.org/gnu/glibc/${v_glibc}.tar.xz" # Linux 5.9 and GCC 12). Building the URL table at parse time would bake
[libxcrypt]="https://github.com/besser82/libxcrypt/releases/download/v${v_libxcrypt}/libxcrypt-${v_libxcrypt}.tar.xz" # in the default linux-7.1.x / gcc-15.3.0 URLs and ignore the per-target
[gmp]="https://ftp.gnu.org/gnu/gmp/${v_gmp}.tar.xz" # overrides, causing f_decompress() to look for the wrong tarball later.
[mpfr]="https://ftp.gnu.org/gnu/mpfr/${v_mpfr}.tar.xz" #
[mpc]="https://ftp.gnu.org/gnu/mpc/${v_mpc}.tar.xz" # The kernel URL path component (v5.x / v6.x / v7.x) is derived from the
[musl]="https://musl.libc.org/releases/${v_musl}.tar.gz" # major version of v_linux so the script works across kernel series without
) # manual URL edits.
declare -A A_SRC_URL=()
function _archive_sane() { function _archive_sane() {
# Quick integrity test: returns 0 if archive is valid, 1 if corrupt. # Quick integrity test: returns 0 if archive is valid, 1 if corrupt.
@ -664,6 +751,24 @@ function f_download() {
cd "${SOURCE_CACHE}" cd "${SOURCE_CACHE}"
echo ">> [ACQUIRE] Downloading upstream source tarballs..." echo ">> [ACQUIRE] Downloading upstream source tarballs..."
# Build the URL table NOW (after any per-target overrides in
# _configure_from_target have been applied). The kernel URL path
# component (v5.x / v6.x / v7.x) is derived from the major version
# of v_linux so tile targets (linux-5.4.x) and mainline targets
# (linux-7.1.x) both resolve correctly.
local _linux_major="${v_linux#linux-}"
_linux_major="${_linux_major%%.*}"
A_SRC_URL[binutils]="https://ftp.gnu.org/gnu/binutils/${v_binutils}.tar.xz"
A_SRC_URL[linux]="https://cdn.kernel.org/pub/linux/kernel/v${_linux_major}.x/${v_linux}.tar.xz"
A_SRC_URL[linux-headers]="https://cdn.kernel.org/pub/linux/kernel/v${_linux_major}.x/${v_linux_headers}.tar.xz"
A_SRC_URL[gcc]="https://ftp.gnu.org/gnu/gcc/${v_gcc}/${v_gcc}.tar.xz"
A_SRC_URL[glibc]="https://ftp.gnu.org/gnu/glibc/${v_glibc}.tar.xz"
A_SRC_URL[libxcrypt]="https://github.com/besser82/libxcrypt/releases/download/v${v_libxcrypt}/libxcrypt-${v_libxcrypt}.tar.xz"
A_SRC_URL[gmp]="https://ftp.gnu.org/gnu/gmp/${v_gmp}.tar.xz"
A_SRC_URL[mpfr]="https://ftp.gnu.org/gnu/mpfr/${v_mpfr}.tar.xz"
A_SRC_URL[mpc]="https://ftp.gnu.org/gnu/mpc/${v_mpc}.tar.xz"
A_SRC_URL[musl]="https://musl.libc.org/releases/${v_musl}.tar.gz"
for key in "${!A_SRC_URL[@]}"; do for key in "${!A_SRC_URL[@]}"; do
local url="${A_SRC_URL[$key]}" local url="${A_SRC_URL[$key]}"
local file="${url##*/}" local file="${url##*/}"
@ -885,14 +990,23 @@ function f_gcc_p1() {
mkdir -p "${SOURCES_ACTIVE}/${v_gcc}-phase1" && cd "${SOURCES_ACTIVE}/${v_gcc}-phase1" mkdir -p "${SOURCES_ACTIVE}/${v_gcc}-phase1" && cd "${SOURCES_ACTIVE}/${v_gcc}-phase1"
# Base configure flags common to all targets # Base configure flags common to all targets.
#
# --with-cpu is x86_64-only — see f_gcc_p2() for the same rationale.
# ARM, MIPS, and Tile-Gx carry their per-target --with-* hints via
# BTC_T_GCC_EXTRA (set in the registry) which is appended below.
local gcc_p1_arch_flags=""
if [[ "${BTC_T_ARCH}" == "x86_64" ]]; then
gcc_p1_arch_flags="--with-cpu=${BTC_T_MARCH}"
fi
local gcc_base="--target=${TARGET} \ local gcc_base="--target=${TARGET} \
--prefix=${NEWROOT} \ --prefix=${NEWROOT} \
--with-sysroot=${NEWROOT} \ --with-sysroot=${NEWROOT} \
--with-newlib \ --with-newlib \
--without-headers \ --without-headers \
--with-arch=${BTC_T_MARCH} \ --with-arch=${BTC_T_MARCH} \
--with-cpu=${BTC_T_MARCH} \ ${gcc_p1_arch_flags} \
--enable-default-pie \ --enable-default-pie \
--enable-default-ssp \ --enable-default-ssp \
--disable-nls \ --disable-nls \
@ -1063,6 +1177,7 @@ function f_gcc_p2() {
mkdir -p "${SOURCES_ACTIVE}/${v_gcc}-phase2" && cd "${SOURCES_ACTIVE}/${v_gcc}-phase2" mkdir -p "${SOURCES_ACTIVE}/${v_gcc}-phase2" && cd "${SOURCES_ACTIVE}/${v_gcc}-phase2"
# Stage 2 final compiler — runs on BUILD host, targets the sysroot. # Stage 2 final compiler — runs on BUILD host, targets the sysroot.
#
# CRITICAL: --host must be ${HOST_ARCH} (the machine the compiler # CRITICAL: --host must be ${HOST_ARCH} (the machine the compiler
# executes on), NOT ${TARGET}. --target specifies what architecture # executes on), NOT ${TARGET}. --target specifies what architecture
# the produced compiler generates code for. Using --host=${TARGET} # the produced compiler generates code for. Using --host=${TARGET}
@ -1072,6 +1187,24 @@ function f_gcc_p2() {
# --with-sysroot and --with-headers are in the base because both # --with-sysroot and --with-headers are in the base because both
# glibc and musl targets need the compiler to find C library # glibc and musl targets need the compiler to find C library
# headers and runtime in the sysroot. # headers and runtime in the sysroot.
#
# --with-cpu is x86_64-only. ARM, MIPS, and Tile-Gx configure
# backends reject --with-cpu=<march> (they want --with-cpu=<cpu>
# e.g. cortex-a9, not the architecture name). Those targets
# already carry --with-arch=<arch> via BTC_T_GCC_EXTRA from the
# registry, so we omit --with-cpu entirely for non-x86_64 to
# avoid "unrecognized argument" failures during configure.
#
# LDFLAGS=-Wl,-rpath,/usr/lib/../lib matches the LFS final-GCC
# recipe. Without it, the freshly-built ${TARGET}-gcc may fail
# at runtime to locate libc.so in the sysroot, which surfaces as
# "cannot find libc.so.6" during the Stage 2 libstdc++ configure
# link tests and aborts Phase 2.
local gcc_p2_arch_flags=""
if [[ "${BTC_T_ARCH}" == "x86_64" ]]; then
gcc_p2_arch_flags="--with-cpu=${BTC_T_MARCH}"
fi
local gcc_p2_base="--prefix=/usr \ local gcc_p2_base="--prefix=/usr \
--build=${HOST_ARCH} \ --build=${HOST_ARCH} \
--host=${HOST_ARCH} \ --host=${HOST_ARCH} \
@ -1079,21 +1212,35 @@ function f_gcc_p2() {
--with-sysroot=${NEWROOT} \ --with-sysroot=${NEWROOT} \
--with-headers=${NEWROOT}/usr/include \ --with-headers=${NEWROOT}/usr/include \
--with-arch=${BTC_T_MARCH} \ --with-arch=${BTC_T_MARCH} \
--with-cpu=${BTC_T_MARCH} \ ${gcc_p2_arch_flags} \
--enable-languages=c,c++ \ --enable-languages=c,c++ \
--enable-default-pie \ --enable-default-pie \
--enable-default-ssp \ --enable-default-ssp \
--enable-threads=posix \ --enable-threads=posix \
--disable-bootstrap" --disable-bootstrap \
LDFLAGS=\"-Wl,-rpath,/usr/lib/../lib\""
# Per-C-library configure adjustments # Per-C-library configure adjustments.
#
# glibc targets: pull in the LFS final-GCC flags so libstdc++
# configure finds glibc locale support (--enable-clocale=gnu),
# uses the __cxa_atexit path for static destructors (required by
# the C++ ABI), and skips PCH generation (--disable-libstdcxx-pch,
# which would otherwise require running the just-built cross-gcc
# to emit a .gch file — impossible during a cross build where we
# cannot execute target binaries).
#
# musl targets: point GCC at the musl library path and inherit
# per-target --with-* hints (arch, fpu, abi, ...) from BTC_T_GCC_EXTRA.
local gcc_p2_clib="" local gcc_p2_clib=""
case "${BTC_T_CLIB}" in case "${BTC_T_CLIB}" in
glibc) glibc)
gcc_p2_clib="--disable-multilib" gcc_p2_clib="--disable-multilib \
--enable-clocale=gnu \
--enable-__cxa_atexit \
--disable-libstdcxx-pch"
;; ;;
musl) musl)
# musl targets: also point GCC at the musl library path.
gcc_p2_clib="--disable-multilib \ gcc_p2_clib="--disable-multilib \
--with-libs=${NEWROOT}/usr/lib \ --with-libs=${NEWROOT}/usr/lib \
${BTC_T_GCC_EXTRA}" ${BTC_T_GCC_EXTRA}"
@ -1105,6 +1252,16 @@ function f_gcc_p2() {
f_exec_log "${build_cmd}" "gcc-p2-configure" f_exec_log "${build_cmd}" "gcc-p2-configure"
f_exec_log "make ${v_threads}" "gcc-p2-make" f_exec_log "make ${v_threads}" "gcc-p2-make"
f_exec_log "make DESTDIR=${NEWROOT} install" "gcc-p2-install" f_exec_log "make DESTDIR=${NEWROOT} install" "gcc-p2-install"
# Stage 2 GCC was installed at ${NEWROOT}/usr/bin/${TARGET}-gcc.
# Prepend that to PATH so f_kernel_binary() picks up the final
# compiler instead of the stripped-down Stage 1 cross-gcc at
# ${NEWROOT}/bin/${TARGET}-gcc (which was built with
# --disable-shared --disable-threads --disable-libstdcxx and
# is unsuitable as the production compiler for the kernel and
# any downstream package builds that link against libgcc_s.
export PATH="${NEWROOT}/usr/bin:${PATH}"
echo ">> [PATH] Prepended ${NEWROOT}/usr/bin for Stage 2 GCC."
} }
# --- 8f. Kernel Binary (architecture-aware) --- # --- 8f. Kernel Binary (architecture-aware) ---
@ -1118,25 +1275,82 @@ function f_kernel_binary() {
echo ">> Instantiating Silicon Optimized Monolithic Configuration Matrix for ${BTC_T_ARCH}..." echo ">> Instantiating Silicon Optimized Monolithic Configuration Matrix for ${BTC_T_ARCH}..."
# Select the correct defconfig for the target architecture # Map BTC's target architecture to the Linux kernel's ARCH= name.
# The kernel's ARCH= variable accepts only the canonical family name
# (arm, mips, x86_64, tilegx, ...). Our registry uses "mipsel" to
# distinguish little-endian MIPS from big-endian, but the kernel's
# Makefile treats MIPS endianness as a Kconfig selection, not as a
# separate ARCH. Passing ARCH=mipsel makes the kernel look for
# arch/mipsel/ which does not exist and the build dies at "make
# defconfig". Resolve the canonical name here, once, so every
# downstream make invocation (defconfig, olddefconfig, image, install)
# uses the same value.
local kernel_arch
case "${BTC_T_ARCH}" in
mipsel) kernel_arch="mips" ;;
*) kernel_arch="${BTC_T_ARCH}" ;;
esac
# LD_LIBRARY_PATH for Stage 2 GCC's C++ runtime (libstdc++.so, libgcc_s.so).
# See the long comment near the kernel-bin-make invocation below for the
# full rationale. Set it BEFORE defconfig because defconfig also probes
# the cross-compiler (`${CROSS_COMPILE}gcc -print-file-name=...`) and a
# broken probe produces an empty .config — exactly the 527-byte defconfig
# output symptom that aborts the build silently.
local gcc_ver="${v_gcc#gcc-}"
local stage2_libpath="${NEWROOT}/usr/lib:${NEWROOT}/lib"
local gcc_libdir="${NEWROOT}/usr/lib/gcc/${TARGET}/${gcc_ver}"
if [[ -d "${gcc_libdir}" ]]; then
stage2_libpath="${stage2_libpath}:${gcc_libdir}"
[[ -d "${gcc_libdir}/32" ]] && stage2_libpath="${stage2_libpath}:${gcc_libdir}/32"
fi
export LD_LIBRARY_PATH="${stage2_libpath}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
echo ">> [LD_LIBRARY_PATH] ${LD_LIBRARY_PATH}"
# Select the correct defconfig for the target architecture.
# These are wrapped in f_exec_log() so the defconfig output is
# captured in ${LOGS}/kernel-bin-defconfig.log — matching every
# other build step in the pipeline. Without logging, a defconfig
# failure surfaces as a bare error on stdout with no archived
# record for post-mortem.
local defconfig_target
case "${BTC_T_ARCH}" in case "${BTC_T_ARCH}" in
arm) arm)
# Multi-v7 is the universal ARMv7 defconfig (covers most Cortex-A SoCs) # multi_v7 is the universal ARMv7 defconfig (covers most Cortex-A SoCs)
make multi_v7_defconfig defconfig_target="multi_v7_defconfig"
;; ;;
mipsel) mipsel)
# MALTA is the reference MIPS32 platform # MALTA is the reference MIPS32 platform
make malta_defconfig defconfig_target="malta_defconfig"
;; ;;
tilegx) tilegx)
# Tile-Gx has its own defconfig # Tile-Gx has its own defconfig; fall back to defconfig for
make tilegx_defconfig 2>/dev/null || make defconfig # kernels where tilegx_defconfig was already removed.
defconfig_target="tilegx_defconfig"
;; ;;
*) *)
make defconfig defconfig_target="defconfig"
;; ;;
esac esac
# First attempt: the target-specific defconfig.
# For tilegx on kernels that removed tilegx_defconfig, fall back to
# the generic defconfig (which still respects ARCH=tilegx).
#
# The `if !` form suspends `set -e` for the condition test so a
# failed defconfig doesn't abort the whole script before we can
# try the fallback. For non-tile targets, a defconfig failure
# is a hard error and we return 1.
if ! f_exec_log "make ARCH=${kernel_arch} ${defconfig_target}" "kernel-bin-defconfig"; then
if [[ "${BTC_T_ARCH}" == "tilegx" ]]; then
echo ">> [WARN] ${defconfig_target} not available in ${kernel_src}; falling back to defconfig."
f_exec_log "make ARCH=${kernel_arch} defconfig" "kernel-bin-defconfig"
else
echo ">> [ERROR] ${defconfig_target} failed for ${kernel_src}."
return 1
fi
fi
# Inject Custom Enterprise Swarm Labels & Architecture Parameters # Inject Custom Enterprise Swarm Labels & Architecture Parameters
sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-dcosnet-${SYS_LABEL}\"/" .config sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-dcosnet-${SYS_LABEL}\"/" .config
@ -1146,8 +1360,15 @@ function f_kernel_binary() {
echo "CONFIG_KALLSYMS=n" >> .config echo "CONFIG_KALLSYMS=n" >> .config
echo "CONFIG_DEBUG_FS=n" >> .config echo "CONFIG_DEBUG_FS=n" >> .config
# Cross-compile kernel for non-x86_64 targets # Cross-compile kernel using the canonical ARCH= name and the
local kernel_make_vars="ARCH=${BTC_T_ARCH} CROSS_COMPILE=${TARGET}-" # Stage 2 cross-compiler (now on PATH after f_gcc_p2()).
#
# LD_LIBRARY_PATH was already set up earlier in this function (before
# the defconfig call) so the loader finds Stage 2's libstdc++.so /
# libgcc_s.so first. See the comment near the top of this function
# for the full rationale.
local kernel_make_vars="ARCH=${kernel_arch} CROSS_COMPILE=${TARGET}-"
f_exec_log "make ${kernel_make_vars} olddefconfig" "kernel-bin-config-merge" f_exec_log "make ${kernel_make_vars} olddefconfig" "kernel-bin-config-merge"
f_exec_log "make ${v_threads} ${kernel_make_vars} LOCALVERSION=-dcosnet-${SYS_LABEL}" "kernel-bin-make" f_exec_log "make ${v_threads} ${kernel_make_vars} LOCALVERSION=-dcosnet-${SYS_LABEL}" "kernel-bin-make"
@ -1197,7 +1418,7 @@ function f_package() {
cat > "${manifest}" << MANIFEST_EOF cat > "${manifest}" << MANIFEST_EOF
{ {
"btc_version": "0.4.0", "btc_version": "${BTC_VERSION}",
"mode": "${mode_label}", "mode": "${mode_label}",
"cross_mode": ${CROSS_MODE}, "cross_mode": ${CROSS_MODE},
"sys_label": "${SYS_LABEL}", "sys_label": "${SYS_LABEL}",
@ -1279,7 +1500,7 @@ function f_main() {
exit 0 exit 0
;; ;;
--help|-h) --help|-h)
echo "BTC-0.4.0.sh - Cross-Compilation Build Tool Chain" echo "BTC-${BTC_VERSION} - Cross-Compilation Build Tool Chain"
echo "" echo ""
echo "Usage: BTC.sh [TARGET_ID | --native | --list | --list-json]" echo "Usage: BTC.sh [TARGET_ID | --native | --list | --list-json]"
echo " BTC.sh --tpm-seal [TARGET_ID]" echo " BTC.sh --tpm-seal [TARGET_ID]"

56
NOTES.md Executable file → Normal file
View File

@ -23,8 +23,8 @@ produce target binaries — every target has its own dedicated cross-toolchain.
### Table-Driven Target Registry ### Table-Driven Target Registry
All 19 targets are defined in a single associative array All 22 targets are defined in a single associative array
(`BTC_TARGETS[]`). Each entry specifies ten fields in pipe-delimited format: (`BTC_TARGETS[]`). Each entry specifies eleven fields in pipe-delimited format:
arch|multilib_arch|march|ISA|abi|libc|endian|family|description|min_kernel|gcc_extra arch|multilib_arch|march|ISA|abi|libc|endian|family|description|min_kernel|gcc_extra
@ -44,11 +44,16 @@ toolchain is packaged into its golden image tarball.
Every binary produced by a BTC-built toolchain carries two immutable Every binary produced by a BTC-built toolchain carries two immutable
identifiers: identifiers:
1. **ELF `.note.BTC` section** — note name "BTC", note type 0xB7C (vendor), 1. **ELF `.note.BTC` section** — note name "DCOSNET", note type 0xB7C
containing a pipe-delimited string with org, version, target, march, ISA, (vendor-specific), containing a pipe-delimited string with org, kernel
and a bare hex SHA-256 hash of the source tarball. version, target ID, march, ISA, sys_label, build stage, signature tier,
2. **Extended attributes (xattr)** — the same stamp data is written to and the active signature token.
`user.btc.stamp` on the binary file. 2. **Extended attributes (xattr)** — four attributes written to the
binary file:
- `user.btc.identity` — `BTC-<SYS_LABEL>-<v_linux>-<sig_tier>`
- `user.btc.hash` — SHA-256 of the stamped binary
- `user.btc.sig.tier` — `poly` | `tpm` | `cluster`
- `user.btc.sig.token` — the active signature token
These stamps allow any binary to be traced back to the exact build environment, These stamps allow any binary to be traced back to the exact build environment,
toolchain version, and source tree that produced it. toolchain version, and source tree that produced it.
@ -77,19 +82,29 @@ determines the optimization flags passed to GCC:
| TILE | tilegx | (arch set per target) | | TILE | tilegx | (arch set per target) |
The SSE4_2 tier exists because Intel Atom and AMD APU low-power cores lack The SSE4_2 tier exists because Intel Atom and AMD APU low-power cores lack
AVX support. GCC is configured with `--with-arch=<march>` and AVX support. GCC is configured with `--with-arch=<march>` in both Stage 1
`--with-cpu=<march>` in both Stage 1 and Stage 2 to ensure the cross-compiler and Stage 2 to ensure the cross-compiler defaults to the correct target
defaults to the correct target microarchitecture. microarchitecture. `--with-cpu=<march>` is added for x86_64 targets only —
ARM, MIPS, and TILE backends reject `--with-cpu=<arch>` (they want a CPU
name like `cortex-a9`, not an architecture name like `armv7-a`) and instead
carry their per-target `--with-*` hints via `BTC_T_GCC_EXTRA`.
## LFS Base Standards ## LFS Base Standards
BTC.sh follows Linux From Scratch 13.0 stable (released 2024-09-01): BTC.sh follows Linux From Scratch 13.0 stable (released 2024-09-01), bumped
forward to the latest point releases actually downloadable from upstream
mirrors as of v0.4.2:
- Binutils 2.46 - Binutils 2.46.1 (LFS 13.0 ships 2.46)
- GCC 14.2.0 - GCC 15.3.0 (LFS 13.0 ships 14.2.0; bumped to 15.3.0 for znver4 /
- Glibc 2.41 sierraforest march support and GCC 15 stricter
- musl 1.2.5 const-correctness — requires `-Wno-error` for
- Linux kernel headers (matched to target `min_kernel`) libxcrypt 4.5.2, which `f_libxcrypt()` already passes)
- Glibc 2.43 (LFS 13.0 ships 2.41)
- musl 1.2.6 (LFS 13.0 ships 1.2.5)
- Linux 7.1.7 (latest 7.1.x point release; LFS 13.0 ships 6.10)
- Tile-Gx override: Linux 5.4.302 LTS + GCC 10.3.0 (mainline dropped
tile in Linux 5.9 / GCC 12)
## C Library Selection Rationale ## C Library Selection Rationale
@ -115,9 +130,12 @@ attribution purposes and are not directly incorporated into the script:
## Source Cache Policy ## Source Cache Policy
BTC.sh caches all downloaded source tarballs locally to avoid placing BTC.sh caches all downloaded source tarballs locally under `/opt/BTC/src/`
unnecessary load on upstream hosting infrastructure. Automated bulk downloads to avoid placing unnecessary load on upstream hosting infrastructure.
should be rate-limited and sources retained after initial fetch. Automated bulk downloads should be rate-limited and sources retained after
initial fetch. The cache is reused across builds; the integrity check in
`_archive_sane()` automatically discards and re-fetches corrupted archives
on the next run.
## License ## License

36
btc-quickstart.md Executable file → Normal file
View File

@ -1,6 +1,6 @@
# BTC Quickstart — Version 0.4.1 # BTC Quickstart — Version 0.4.2
A guide to building your first cross-toolchain with BTC.sh . A guide to building your first cross-toolchain with BTC.sh.
## 1. Prerequisites ## 1. Prerequisites
@ -19,17 +19,17 @@ executes in four phases:
available RAM to prevent LTO thrashing. The target registry is loaded. available RAM to prevent LTO thrashing. The target registry is loaded.
2. **Setup** — A volatile cleanroom (ramfs) is provisioned at the configured 2. **Setup** — A volatile cleanroom (ramfs) is provisioned at the configured
mount point. Source tarballs are verified against their SHA-256 checksums. mount point. Source tarballs are verified against their SHA-256 checksums.
3. **STOP USING THIS WORD >>>Build** — Core components are built sequentially: 3. **Build** — Core components are built sequentially:
Binutils → Kernel Headers → GCC Stage 1 → C Library (glibc or musl) → GCC Stage 2 → Kernel. Binutils → Kernel Headers → GCC Stage 1 → C Library (glibc or musl) → GCC Stage 2 → Kernel.
Both GCC stages are configured with `--with-arch=<march>` and `--with-cpu=<march>` Both GCC stages are configured with `--with-arch=<march>` (and `--with-cpu=<march>`
to default to the target microarchitecture. for x86_64 targets only) to default to the target microarchitecture.
4. **Package** — The resulting cross-toolchain is compressed into a golden 4. **Package** — The resulting cross-toolchain is compressed into a golden
image tarball. A manifest JSON sidecar and forensic ELF stamp are applied. image tarball. A manifest JSON sidecar and forensic ELF stamp are applied.
## 3. Build a Cross-Toolchain ## 3. Build a Cross-Toolchain
```bash ```bash
# List all 19 available targets # List all 22 available targets
sudo ./BTC.sh --list sudo ./BTC.sh --list
# Build a cross-toolchain for AMD Zen3 (Ryzen 5000 / EPYC Milan) # Build a cross-toolchain for AMD Zen3 (Ryzen 5000 / EPYC Milan)
@ -48,18 +48,20 @@ sudo ./BTC.sh --native
## 4. Verify the Golden Image ## 4. Verify the Golden Image
After a successful build, the golden image tarball and its manifest are written After a successful build, the golden image tarball and its manifest are written
to `/opt/BTC/releases/`: directly to `/opt/BTC/` (the manifest uses the `${SYS_LABEL}-manifest.json`
naming convention; the golden image uses `${SYS_LABEL}-toolchain-golden.tar.xz`):
```bash ```bash
# List available golden images # List available golden images
ls -la /opt/BTC/releases/ ls -la /opt/BTC/*-toolchain-golden.tar.xz /opt/BTC/*-manifest.json
# Inspect the manifest # Inspect the manifest
cat /opt/BTC/releases/DCOSNET-amd-znver3-AVX2-CROSS-toolchain-manifest.json cat /opt/BTC/DCOSNET-AMD-ZNVER3-AVX2-CROSS-manifest.json
``` ```
The manifest contains structured metadata: target ID, architecture, C library, The manifest contains structured metadata: target ID, architecture, C library,
microarchitecture, ISA tier, cross-compiler triple, and build timestamps. microarchitecture, ISA tier, cross-compiler triple, signature tier, and build
configuration flags.
## 5. Forensic Stamp Verification ## 5. Forensic Stamp Verification
@ -68,10 +70,10 @@ section. Verify it:
```bash ```bash
# Read the ELF note # Read the ELF note
readelf -n /path/to/binary | grep -A5 BTC readelf -n /path/to/binary | grep -A5 DCOSNET
# Read the xattr stamp # Read the four xattr stamps
getfattr -d user.btc.stamp /path/to/binary getfattr -d user.btc.identity,user.btc.hash,user.btc.sig.tier,user.btc.sig.token /path/to/binary
``` ```
## 6. Integration with Sorcery-Go and Fester ## 6. Integration with Sorcery-Go and Fester
@ -84,7 +86,7 @@ LDFLAGS), and verifies stamps on build outputs.
## 7. Source Cache ## 7. Source Cache
BTC.sh caches downloaded source tarballs in `/opt/BTC/sources/`. If a tarball BTC.sh caches downloaded source tarballs in `/opt/BTC/src/`. If a tarball
is already present and its checksum matches, it is not re-downloaded. Keep the is already present and its integrity check passes, it is not re-downloaded.
cache directory intact between builds to avoid unnecessary load on upstream Keep the cache directory intact between builds to avoid unnecessary load on
mirrors. upstream mirrors.

131
cleanup.sh Executable file → Normal file
View File

@ -1,11 +1,124 @@
#!/bin/bash #!/bin/bash
# On target: # cleanup.sh — post-build janitor for BTC.sh
cd /opt/BTC #
rm -rf BTC-0.4.1 # Removes the volatile cleanroom, unmounts the ramfs, and re-stages a fresh
umount -l /usr/src 2>/dev/null || true # BTC.sh copy under /opt/BTC/BTC-<version>/ ready to be invoked.
rm -rf /usr/src/DCOSNET-HASWELL-AVX2-LTO-cleanroom #
rm -rf logs/DCOSNET-HASWELL-AVX2-LTO # Usage:
# ./cleanup.sh # uses BTC_VERSION from BTC.sh auto-detect
# ./cleanup.sh <target_id> # e.g. ./cleanup.sh haswell
# ./cleanup.sh <target_id> <ver> # e.g. ./cleanup.sh haswell 0.4.2
#
# Defaults:
# target_id = haswell (matches the original cleanup.sh behavior)
# version = read from BTC.sh (falls back to 0.4.2)
#
# NOTE: this script must be run on the target machine AFTER the BTC.sh build
# host has produced the golden image. It does NOT rebuild anything — it
# just clears the working state and re-stages the next BTC.sh copy.
set -euo pipefail
BTC_ARCHIVE="/opt/BTC"
SOURCES_ACTIVE="/usr/src"
# --- Resolve target_id argument -----------------------------------------
# The original cleanup.sh hardcoded DCOSNET-HASWELL-AVX2-LTO. We default
# to the same target so behavior is unchanged for users who run the script
# with no arguments, but expose the full SYS_LABEL derivation so any of the
# 22 registered targets can be cleaned up by passing its target_id.
TARGET_ID="${1:-haswell}"
# --- Resolve BTC_VERSION -------------------------------------------------
# Prefer an explicit second arg, otherwise parse it out of the BTC.sh
# sitting next to this script so the cleanup tracks the deployed version
# automatically. Falls back to 0.4.2 if neither is available.
BTC_VERSION="${2:-}"
if [[ -z "${BTC_VERSION}" ]]; then
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BTC_SH="${SCRIPT_DIR}/BTC.sh"
if [[ -f "${BTC_SH}" ]]; then
BTC_VERSION="$(grep -m1 -E '^readonly BTC_VERSION=' "${BTC_SH}" \
| sed -E 's/^readonly BTC_VERSION="([^"]+)".*$/\1/' || true)"
fi
fi
BTC_VERSION="${BTC_VERSION:-0.4.2}"
# --- Derive SYS_LABEL the same way BTC.sh does ---------------------------
# For native x86_64 builds BTC.sh produces SYS_LABEL=DCOSNET-<ID>-<ISA>-LTO.
# For cross builds it produces SYS_LABEL=DCOSNET-<FAMILY>-<ID>-<ISA>-CROSS.
# The cleanup needs to match the build's actual label, so we use the same
# uppercase + family/ISA tags. ISA is read from BTC_TARGETS[] if BTC.sh
# is reachable; otherwise we conservatively probe by family.
FAMILY=""
case "${TARGET_ID}" in
# AVX512 Intel HEDT/Server (must come before the AVX2 catch-all)
skylake-x|skylake-server)
FAMILY="intel"; ISA_TAG="AVX512" ;;
# AVX2 Intel HEDT/Server
haswell|haswell-ep|broadwell|broadwell-ep|skylake)
FAMILY="intel"; ISA_TAG="AVX2" ;;
# AVX512 AMD
znver4)
FAMILY="amd"; ISA_TAG="AVX512" ;;
# AVX2 AMD
znver1|znver2|znver3)
FAMILY="amd"; ISA_TAG="AVX2" ;;
apu-zn1|apu-zn2|apu-zn3|apu-zn4)
FAMILY="amd-apu"; ISA_TAG="AVX2" ;;
# SSE4_2 Intel Atom
atom-silvermont|atom-goldmont|atom-tremont|atom-sierraforest)
FAMILY="atom"; ISA_TAG="SSE4_2" ;;
# Embedded
mipselr2)
FAMILY="mips"; ISA_TAG="MIPS32" ;;
armv7)
FAMILY="arm"; ISA_TAG="NEON" ;;
tilegx)
FAMILY="tile"; ISA_TAG="TILE" ;;
*)
echo ">> [ERROR] Unknown target_id: ${TARGET_ID}"
echo ">> Run 'BTC.sh --list' to see registered targets."
exit 1
;;
esac
# Mirror BTC.sh's native-vs-cross SYS_LABEL rule:
# native -> DCOSNET-<ID>-<ISA>-LTO
# cross -> DCOSNET-<FAMILY>-<ID>-<ISA>-CROSS
# x86_64 targets are typically run native; everything else is cross.
if [[ "${FAMILY}" == "intel" || "${FAMILY}" == "amd" || "${FAMILY}" == "amd-apu" || "${FAMILY}" == "atom" ]]; then
SYS_LABEL="DCOSNET-${TARGET_ID^^}-${ISA_TAG}-LTO"
else
SYS_LABEL="DCOSNET-${FAMILY^^}-${TARGET_ID^^}-${ISA_TAG}-CROSS"
fi
echo ">> [CLEANUP] target_id=${TARGET_ID} version=${BTC_VERSION} sys_label=${SYS_LABEL}"
# --- Tear down the previous build state ----------------------------------
cd "${BTC_ARCHIVE}"
rm -rf "BTC-${BTC_VERSION}"
# Lazy-unmount the ramfs cleanroom (in case a previous run crashed mid-build).
umount -l "${SOURCES_ACTIVE}" 2>/dev/null || true
# Remove the target-specific cleanroom directory and its logs.
rm -rf "${SOURCES_ACTIVE}/${SYS_LABEL}-cleanroom"
rm -rf "${BTC_ARCHIVE}/logs/${SYS_LABEL}"
# --- Re-stage the next BTC.sh copy ---------------------------------------
# Upload the fixed BTC.sh from download/, then: # Upload the fixed BTC.sh from download/, then:
mkdir -p BTC-0.4.1 mkdir -p "BTC-${BTC_VERSION}"
cp BTC.sh BTC-0.4.1/ if [[ -f "${BTC_ARCHIVE}/BTC.sh" ]]; then
cd BTC-0.4.1 && chmod +x BTC.sh && echo "btc ready in /opt/BTC/BTC-0.4.1/" #./BTC.sh cp "${BTC_ARCHIVE}/BTC.sh" "BTC-${BTC_VERSION}/"
elif [[ -f "./BTC.sh" ]]; then
cp "./BTC.sh" "BTC-${BTC_VERSION}/"
else
echo ">> [WARN] No BTC.sh found to stage under BTC-${BTC_VERSION}/."
echo ">> Drop BTC.sh into ${BTC_ARCHIVE}/ or ${PWD}/ and re-run."
fi
cd "BTC-${BTC_VERSION}" && chmod +x BTC.sh
echo ">> [READY] btc staged at ${BTC_ARCHIVE}/BTC-${BTC_VERSION}/BTC.sh"
echo ">> Target label cleaned: ${SYS_LABEL}"
echo ">> Invoke with: ./BTC.sh ${TARGET_ID}"