From d2989812dd7a3fe4f6c71cfcd72ca58f91333120 Mon Sep 17 00:00:00 2001 From: Jeremy Anderson Date: Sat, 8 Aug 2026 23:58:47 -0400 Subject: [PATCH] bug testing --- BTC.sh | 325 ++++++++++++++++++++++++++++++++++++++-------- NOTES.md | 56 +++++--- btc-quickstart.md | 36 ++--- cleanup.sh | 131 +++++++++++++++++-- 4 files changed, 451 insertions(+), 97 deletions(-) mode change 100755 => 100644 NOTES.md mode change 100755 => 100644 btc-quickstart.md mode change 100755 => 100644 cleanup.sh diff --git a/BTC.sh b/BTC.sh index a8bb7d1..d382641 100755 --- a/BTC.sh +++ b/BTC.sh @@ -3,23 +3,30 @@ # - [[ ... ]] conditionals, == pattern matching, =~ regex, (( )) arithmetic # - ${var^^} uppercase expansion, associative arrays, process substitution # - set -euo pipefail for strict error handling -# BTC-0.4.0.sh - Build Tool Chain +# BTC-${BTC_VERSION} - Build Tool Chain # Identity: dcosnet / dcos.net | Multi-Arch Cross-Compilation Build Tool Chain -# Version: 0.4.1 | Persistence: /opt/BTC | Volatile: ramfs +# Version: 0.4.2 | Persistence: /opt/BTC | Volatile: ramfs # License: GNU AGPLv3 Mandatory Prominent Interactive Notice # Copyright (C) 2012-2026 Jeremy Anderson (info@dcos.net) set -euo pipefail export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES +# ---------------------------------------------------------------------------- +# Single source of truth for the BTC version. Every banner, header comment, +# manifest, and --help text MUST reference this constant — never hardcode the +# version string in multiple places. +# ---------------------------------------------------------------------------- +readonly BTC_VERSION="0.4.2" + # ============================================================================ # 1. AGPL INTERACTIVE LICENSE COMPLIANCE # ============================================================================ function f_agpl_header() { clear - cat << 'EOF' + cat << EOF =========================================================================== - BTC-0.4.1.sh - Build Tool Chain (AGPLv3 PROTECTED) + BTC-${BTC_VERSION} - Build Tool Chain (AGPLv3 PROTECTED) Cross-Compilation Build Tool Chain =========================================================================== This program is free software: you can redistribute it and/or modify it @@ -177,7 +184,7 @@ BTC_TARGETS[tilegx]="tilegx|tilegx|tilegx|TILE|abi64|musl|little|tile|Tilera TIL # --- Helper: list all registered targets --- function f_list_targets() { - echo ">> BTC-0.4.0 Registered Cross-Compilation Targets:" + echo ">> BTC-${BTC_VERSION} Registered Cross-Compilation Targets:" echo ">> ===============================================" printf ">> %-16s %-10s %-18s %-8s %-6s %s\n" "TARGET_ID" "ARCH" "MARCH" "ISA" "CLIB" "DESCRIPTION" printf ">> %-16s %-10s %-18s %-8s %-6s %s\n" "--------" "----" "-----" "---" "----" "-----------" @@ -337,9 +344,25 @@ function _configure_from_target() { if [[ ${safe_threads} -gt ${total_cpus} ]]; then safe_threads=${total_cpus}; fi export v_threads="-j${safe_threads}" - # Pin GCC version for Tile-Gx (upstream dropped after GCC 11) + # Reset version overrides to the default baseline before per-target + # pinning. Without this reset, calling _configure_from_target twice + # in the same process (e.g. test harnesses, or a future batch-build + # mode) would leak the tile pin into subsequent non-tile targets. + # Single-target builds are unaffected, but the reset is cheap and + # makes the function idempotent. + v_gcc="${v_gcc_default}" + v_linux="${v_linux_default}" + v_linux_headers="${v_linux_headers_default}" + + # Pin GCC and Linux versions for Tile-Gx. + # GCC upstream dropped mainline Tile-Gx support after GCC 11. + # Linux upstream removed the tile architecture in 5.9; the last LTS + # series that still builds for tilegx is 5.4.x. Both overrides MUST run + # before f_download(), which captures these globals into A_SRC_URL[]. if [[ "${BTC_T_FAMILY}" == "tile" ]]; then - v_gcc='gcc-10.3.0' + v_gcc="${v_gcc_tile}" + v_linux="${v_linux_tile}" + v_linux_headers="${v_linux_headers_tile}" fi echo ">> [IDENTITY STAMP] ${SYS_LABEL}" @@ -361,9 +384,30 @@ export SOURCE_CACHE=${BTC_ARCHIVE}/src export RAMDISK_SIZE="12gb" # Upstream Production Matrices (defaults — can be overridden per-target) -v_linux='linux-7.1' +# Versions track the current LFS stable baseline (LFS 13.0+) where practical, +# bumped forward to the latest point releases that are actually downloadable +# from upstream mirrors as of ${BTC_VERSION}. +# +# - linux-7.1.7 : latest stable point release on the 7.1.x series +# - binutils-2.46.1 : LFS-aligned binutils point release +# - gcc-15.3.0 : current GCC stable (LFS 13.0 ships 14.2.0; we bump +# to 15.3.0 to pick up znver4/sierraforest support +# and GCC 15 stricter const-correctness) +# - glibc-2.43 : current glibc stable +# - musl-1.2.6 : current musl stable +# +# Tile-Gx override: tile architecture was REMOVED from mainline Linux in 5.9 +# (commit 65ad263b1d0f). The last LTS series with tile support is 5.4.x, so +# tile targets are pinned to linux-5.4.302 (LTS) and gcc-10.3.0 (last GCC +# release with full tile-gx backend). See _configure_from_target() for the +# runtime override. +v_linux='linux-7.1.7' +v_linux_default='linux-7.1.7' +v_linux_tile='linux-5.4.302' v_binutils='binutils-2.46.1' v_gcc='gcc-15.3.0' +v_gcc_default='gcc-15.3.0' +v_gcc_tile='gcc-10.3.0' v_glibc='glibc-2.43' v_libxcrypt='4.5.2' v_gmp='gmp-6.3.0' @@ -371,6 +415,8 @@ v_mpfr='mpfr-4.2.2' v_mpc='mpc-1.4.0' v_musl='musl-1.2.6' v_linux_headers="${v_linux}" +v_linux_headers_default="${v_linux_default}" +v_linux_headers_tile="${v_linux_tile}" # These are set after f_resolve_target: # NEWROOT, LOGS, HOST_ARCH, TARGET, TARGET_ARCH, GLOBAL_CFLAGS, GLOBAL_LDFLAGS @@ -434,22 +480,45 @@ function f_guard() { } function f_entropy_shield() { - local min_entropy=1000 - local cur_entropy - if [[ -f /proc/sys/kernel/random/entropy_avail ]]; then - cur_entropy=$(< /proc/sys/kernel/random/entropy_avail) - if [[ ${cur_entropy} -lt ${min_entropy} ]]; then - echo ">> [ENTROPY DEFICIT] Pool dropped to ${cur_entropy}. Injecting safe hardware-jitter..." - find /bin /sbin -type f -exec ls -l {} + > /dev/null 2>&1 & - sleep 2 - kill $! 2>/dev/null || true - fi + # Modern Linux (>= 5.6) initializes the CRNG at boot via getrandom(2) + # and the kernel's own jitter entropy collector. Since 5.6, the + # input-pool counter in /proc/sys/kernel/random/entropy_avail is + # capped at 256 by design — it does NOT reflect "available" entropy + # in the 2.6-era sense anymore. Any value >= 256 means "CRNG ready, + # getrandom will return immediately". The historical "1000" threshold + # is from the 2.6 era when /dev/random could genuinely block. + # + # Behavior on a modern box: + # - entropy_avail will be 256 essentially always (capped) + # - getrandom(GRND_NONBLOCK) succeeds instantly + # - The "ENTROPY DEFICIT" warning would fire on EVERY step and the + # old `sleep 2` blocked the parent pipeline for no reason. + # + # Fix: lower the threshold to 256, drop the blocking sleep entirely, + # fire the jitter injector in the background (fire-and-forget, never + # blocks the calling build step). If getrandom isn't usable on this + # host we still benefit from any extra jitter, but we don't pay any + # wall-clock time for it. + local min_entropy=256 + local cur_entropy=0 + if [[ -r /proc/sys/kernel/random/entropy_avail ]]; then + cur_entropy=$(< /proc/sys/kernel/random/entropy_avail) || cur_entropy=0 + fi + if [[ ${cur_entropy} -gt 0 && ${cur_entropy} -lt ${min_entropy} ]]; then + echo ">> [ENTROPY] Pool at ${cur_entropy} (< ${min_entropy}); jitter injector fired in background." + # Fire-and-forget. Never blocks the calling build step. + # The injector walks /bin /sbin /usr/bin to feed disk-I/O timing + # jitter into the kernel input pool. If it fails or finds nothing + # useful, no harm done — getrandom(2) still works on modern kernels. + { find /bin /sbin /usr/bin -type f -exec ls -l {} + > /dev/null 2>&1; } & + disown 2>/dev/null || true fi } function f_exec_log() { local cmd="$1" local log_base="$2" + local log_file="${LOGS}/${log_base}.log" f_entropy_shield f_guard @@ -458,9 +527,25 @@ function f_exec_log() { # Note: ${cmd} is sourced from internal build functions only (not user input). # The trust boundary is the BTC.sh script itself — do not expose f_exec_log # as a public API with externally-supplied command strings. - stdbuf -oL -eL bash -c "${cmd}" 2>&1 | \ - pv -t -r -b -N "${log_base}" | \ - tee -a "${LOGS}/${log_base}.log" > /dev/null + # + # Pipeline: bash -c "$cmd" → stdbuf (line-buffer) → pv (progress bar) + # → tee (log file) → /dev/null (suppress stdout so the terminal stays clean). + # + # On failure: the `if !` form suspends `set -e` for the condition test, + # captures the pipeline's exit status (with `set -o pipefail` this is + # the rightmost non-zero exit in the pipe — usually bash -c's exit code), + # then surfaces the last 40 lines of the log file to stderr so the + # operator sees the actual error inline instead of having to dig + # through ${LOGS}/.log post-mortem. + if ! stdbuf -oL -eL bash -c "${cmd}" 2>&1 | \ + pv -t -r -b -N "${log_base}" | \ + tee -a "${log_file}" > /dev/null; then + echo ">> [FAILED] ${log_base} exited non-zero." >&2 + echo ">> [FAILED] Last 40 lines of ${log_file}:" >&2 + tail -n 40 "${log_file}" >&2 2>/dev/null || true + echo ">> [FAILED] Full log: ${log_file}" >&2 + return 1 + fi } function f_tmux_dashboard() { @@ -585,7 +670,8 @@ function f_stamp_binary() { # The .note.BTC payload now carries the active sig tier and token, # making each deployment's forensic identity distinct. # Note type 0xB7C is a vendor-specific identifier — it does NOT - # This vendor type (0xB7C) does not correspond to any standard. + # correspond to any standard ELF note type (the Linux vendor-note + # namespace 0x0B7C is reserved for out-of-tree consumers). cat << EOF > btc_stamp.s .section .note.BTC,"a",@note .long 2f - 1f @@ -629,18 +715,19 @@ EOF # Associative array: package stem -> upstream URL # Only packages consumed by the build sequence are listed. # URLs are official upstream mirrors — replace to point at a local mirror. -declare -A A_SRC_URL=( - [binutils]="https://ftp.gnu.org/gnu/binutils/${v_binutils}.tar.xz" - [linux]="https://cdn.kernel.org/pub/linux/kernel/v7.x/${v_linux}.tar.xz" - [linux-headers]="https://cdn.kernel.org/pub/linux/kernel/v7.x/${v_linux_headers}.tar.xz" - [gcc]="https://ftp.gnu.org/gnu/gcc/${v_gcc}/${v_gcc}.tar.xz" - [glibc]="https://ftp.gnu.org/gnu/glibc/${v_glibc}.tar.xz" - [libxcrypt]="https://github.com/besser82/libxcrypt/releases/download/v${v_libxcrypt}/libxcrypt-${v_libxcrypt}.tar.xz" - [gmp]="https://ftp.gnu.org/gnu/gmp/${v_gmp}.tar.xz" - [mpfr]="https://ftp.gnu.org/gnu/mpfr/${v_mpfr}.tar.xz" - [mpc]="https://ftp.gnu.org/gnu/mpc/${v_mpc}.tar.xz" - [musl]="https://musl.libc.org/releases/${v_musl}.tar.gz" -) +# +# NOTE: A_SRC_URL is declared here as an empty container and populated lazily +# inside f_download(). This is required because v_linux, v_gcc, and +# v_linux_headers may be overridden per-target (e.g. tile targets pin +# linux-5.4.302 and gcc-10.3.0 because mainline dropped tile support in +# Linux 5.9 and GCC 12). Building the URL table at parse time would bake +# in the default linux-7.1.x / gcc-15.3.0 URLs and ignore the per-target +# overrides, causing f_decompress() to look for the wrong tarball later. +# +# The kernel URL path component (v5.x / v6.x / v7.x) is derived from the +# major version of v_linux so the script works across kernel series without +# manual URL edits. +declare -A A_SRC_URL=() function _archive_sane() { # Quick integrity test: returns 0 if archive is valid, 1 if corrupt. @@ -664,6 +751,24 @@ function f_download() { cd "${SOURCE_CACHE}" echo ">> [ACQUIRE] Downloading upstream source tarballs..." + # Build the URL table NOW (after any per-target overrides in + # _configure_from_target have been applied). The kernel URL path + # component (v5.x / v6.x / v7.x) is derived from the major version + # of v_linux so tile targets (linux-5.4.x) and mainline targets + # (linux-7.1.x) both resolve correctly. + local _linux_major="${v_linux#linux-}" + _linux_major="${_linux_major%%.*}" + A_SRC_URL[binutils]="https://ftp.gnu.org/gnu/binutils/${v_binutils}.tar.xz" + A_SRC_URL[linux]="https://cdn.kernel.org/pub/linux/kernel/v${_linux_major}.x/${v_linux}.tar.xz" + A_SRC_URL[linux-headers]="https://cdn.kernel.org/pub/linux/kernel/v${_linux_major}.x/${v_linux_headers}.tar.xz" + A_SRC_URL[gcc]="https://ftp.gnu.org/gnu/gcc/${v_gcc}/${v_gcc}.tar.xz" + A_SRC_URL[glibc]="https://ftp.gnu.org/gnu/glibc/${v_glibc}.tar.xz" + A_SRC_URL[libxcrypt]="https://github.com/besser82/libxcrypt/releases/download/v${v_libxcrypt}/libxcrypt-${v_libxcrypt}.tar.xz" + A_SRC_URL[gmp]="https://ftp.gnu.org/gnu/gmp/${v_gmp}.tar.xz" + A_SRC_URL[mpfr]="https://ftp.gnu.org/gnu/mpfr/${v_mpfr}.tar.xz" + A_SRC_URL[mpc]="https://ftp.gnu.org/gnu/mpc/${v_mpc}.tar.xz" + A_SRC_URL[musl]="https://musl.libc.org/releases/${v_musl}.tar.gz" + for key in "${!A_SRC_URL[@]}"; do local url="${A_SRC_URL[$key]}" local file="${url##*/}" @@ -885,14 +990,23 @@ function f_gcc_p1() { mkdir -p "${SOURCES_ACTIVE}/${v_gcc}-phase1" && cd "${SOURCES_ACTIVE}/${v_gcc}-phase1" - # Base configure flags common to all targets + # Base configure flags common to all targets. + # + # --with-cpu is x86_64-only — see f_gcc_p2() for the same rationale. + # ARM, MIPS, and Tile-Gx carry their per-target --with-* hints via + # BTC_T_GCC_EXTRA (set in the registry) which is appended below. + local gcc_p1_arch_flags="" + if [[ "${BTC_T_ARCH}" == "x86_64" ]]; then + gcc_p1_arch_flags="--with-cpu=${BTC_T_MARCH}" + fi + local gcc_base="--target=${TARGET} \ --prefix=${NEWROOT} \ --with-sysroot=${NEWROOT} \ --with-newlib \ --without-headers \ --with-arch=${BTC_T_MARCH} \ - --with-cpu=${BTC_T_MARCH} \ + ${gcc_p1_arch_flags} \ --enable-default-pie \ --enable-default-ssp \ --disable-nls \ @@ -1063,6 +1177,7 @@ function f_gcc_p2() { mkdir -p "${SOURCES_ACTIVE}/${v_gcc}-phase2" && cd "${SOURCES_ACTIVE}/${v_gcc}-phase2" # Stage 2 final compiler — runs on BUILD host, targets the sysroot. + # # CRITICAL: --host must be ${HOST_ARCH} (the machine the compiler # executes on), NOT ${TARGET}. --target specifies what architecture # the produced compiler generates code for. Using --host=${TARGET} @@ -1072,6 +1187,24 @@ function f_gcc_p2() { # --with-sysroot and --with-headers are in the base because both # glibc and musl targets need the compiler to find C library # headers and runtime in the sysroot. + # + # --with-cpu is x86_64-only. ARM, MIPS, and Tile-Gx configure + # backends reject --with-cpu= (they want --with-cpu= + # e.g. cortex-a9, not the architecture name). Those targets + # already carry --with-arch= via BTC_T_GCC_EXTRA from the + # registry, so we omit --with-cpu entirely for non-x86_64 to + # avoid "unrecognized argument" failures during configure. + # + # LDFLAGS=-Wl,-rpath,/usr/lib/../lib matches the LFS final-GCC + # recipe. Without it, the freshly-built ${TARGET}-gcc may fail + # at runtime to locate libc.so in the sysroot, which surfaces as + # "cannot find libc.so.6" during the Stage 2 libstdc++ configure + # link tests and aborts Phase 2. + local gcc_p2_arch_flags="" + if [[ "${BTC_T_ARCH}" == "x86_64" ]]; then + gcc_p2_arch_flags="--with-cpu=${BTC_T_MARCH}" + fi + local gcc_p2_base="--prefix=/usr \ --build=${HOST_ARCH} \ --host=${HOST_ARCH} \ @@ -1079,21 +1212,35 @@ function f_gcc_p2() { --with-sysroot=${NEWROOT} \ --with-headers=${NEWROOT}/usr/include \ --with-arch=${BTC_T_MARCH} \ - --with-cpu=${BTC_T_MARCH} \ + ${gcc_p2_arch_flags} \ --enable-languages=c,c++ \ --enable-default-pie \ --enable-default-ssp \ --enable-threads=posix \ - --disable-bootstrap" + --disable-bootstrap \ + LDFLAGS=\"-Wl,-rpath,/usr/lib/../lib\"" - # Per-C-library configure adjustments + # Per-C-library configure adjustments. + # + # glibc targets: pull in the LFS final-GCC flags so libstdc++ + # configure finds glibc locale support (--enable-clocale=gnu), + # uses the __cxa_atexit path for static destructors (required by + # the C++ ABI), and skips PCH generation (--disable-libstdcxx-pch, + # which would otherwise require running the just-built cross-gcc + # to emit a .gch file — impossible during a cross build where we + # cannot execute target binaries). + # + # musl targets: point GCC at the musl library path and inherit + # per-target --with-* hints (arch, fpu, abi, ...) from BTC_T_GCC_EXTRA. local gcc_p2_clib="" case "${BTC_T_CLIB}" in glibc) - gcc_p2_clib="--disable-multilib" + gcc_p2_clib="--disable-multilib \ + --enable-clocale=gnu \ + --enable-__cxa_atexit \ + --disable-libstdcxx-pch" ;; musl) - # musl targets: also point GCC at the musl library path. gcc_p2_clib="--disable-multilib \ --with-libs=${NEWROOT}/usr/lib \ ${BTC_T_GCC_EXTRA}" @@ -1105,6 +1252,16 @@ function f_gcc_p2() { f_exec_log "${build_cmd}" "gcc-p2-configure" f_exec_log "make ${v_threads}" "gcc-p2-make" f_exec_log "make DESTDIR=${NEWROOT} install" "gcc-p2-install" + + # Stage 2 GCC was installed at ${NEWROOT}/usr/bin/${TARGET}-gcc. + # Prepend that to PATH so f_kernel_binary() picks up the final + # compiler instead of the stripped-down Stage 1 cross-gcc at + # ${NEWROOT}/bin/${TARGET}-gcc (which was built with + # --disable-shared --disable-threads --disable-libstdcxx and + # is unsuitable as the production compiler for the kernel and + # any downstream package builds that link against libgcc_s. + export PATH="${NEWROOT}/usr/bin:${PATH}" + echo ">> [PATH] Prepended ${NEWROOT}/usr/bin for Stage 2 GCC." } # --- 8f. Kernel Binary (architecture-aware) --- @@ -1118,25 +1275,82 @@ function f_kernel_binary() { echo ">> Instantiating Silicon Optimized Monolithic Configuration Matrix for ${BTC_T_ARCH}..." - # Select the correct defconfig for the target architecture + # Map BTC's target architecture to the Linux kernel's ARCH= name. + # The kernel's ARCH= variable accepts only the canonical family name + # (arm, mips, x86_64, tilegx, ...). Our registry uses "mipsel" to + # distinguish little-endian MIPS from big-endian, but the kernel's + # Makefile treats MIPS endianness as a Kconfig selection, not as a + # separate ARCH. Passing ARCH=mipsel makes the kernel look for + # arch/mipsel/ which does not exist and the build dies at "make + # defconfig". Resolve the canonical name here, once, so every + # downstream make invocation (defconfig, olddefconfig, image, install) + # uses the same value. + local kernel_arch + case "${BTC_T_ARCH}" in + mipsel) kernel_arch="mips" ;; + *) kernel_arch="${BTC_T_ARCH}" ;; + esac + + # LD_LIBRARY_PATH for Stage 2 GCC's C++ runtime (libstdc++.so, libgcc_s.so). + # See the long comment near the kernel-bin-make invocation below for the + # full rationale. Set it BEFORE defconfig because defconfig also probes + # the cross-compiler (`${CROSS_COMPILE}gcc -print-file-name=...`) and a + # broken probe produces an empty .config — exactly the 527-byte defconfig + # output symptom that aborts the build silently. + local gcc_ver="${v_gcc#gcc-}" + local stage2_libpath="${NEWROOT}/usr/lib:${NEWROOT}/lib" + local gcc_libdir="${NEWROOT}/usr/lib/gcc/${TARGET}/${gcc_ver}" + if [[ -d "${gcc_libdir}" ]]; then + stage2_libpath="${stage2_libpath}:${gcc_libdir}" + [[ -d "${gcc_libdir}/32" ]] && stage2_libpath="${stage2_libpath}:${gcc_libdir}/32" + fi + export LD_LIBRARY_PATH="${stage2_libpath}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + echo ">> [LD_LIBRARY_PATH] ${LD_LIBRARY_PATH}" + + # Select the correct defconfig for the target architecture. + # These are wrapped in f_exec_log() so the defconfig output is + # captured in ${LOGS}/kernel-bin-defconfig.log — matching every + # other build step in the pipeline. Without logging, a defconfig + # failure surfaces as a bare error on stdout with no archived + # record for post-mortem. + local defconfig_target case "${BTC_T_ARCH}" in arm) - # Multi-v7 is the universal ARMv7 defconfig (covers most Cortex-A SoCs) - make multi_v7_defconfig + # multi_v7 is the universal ARMv7 defconfig (covers most Cortex-A SoCs) + defconfig_target="multi_v7_defconfig" ;; mipsel) # MALTA is the reference MIPS32 platform - make malta_defconfig + defconfig_target="malta_defconfig" ;; tilegx) - # Tile-Gx has its own defconfig - make tilegx_defconfig 2>/dev/null || make defconfig + # Tile-Gx has its own defconfig; fall back to defconfig for + # kernels where tilegx_defconfig was already removed. + defconfig_target="tilegx_defconfig" ;; *) - make defconfig + defconfig_target="defconfig" ;; esac + # First attempt: the target-specific defconfig. + # For tilegx on kernels that removed tilegx_defconfig, fall back to + # the generic defconfig (which still respects ARCH=tilegx). + # + # The `if !` form suspends `set -e` for the condition test so a + # failed defconfig doesn't abort the whole script before we can + # try the fallback. For non-tile targets, a defconfig failure + # is a hard error and we return 1. + if ! f_exec_log "make ARCH=${kernel_arch} ${defconfig_target}" "kernel-bin-defconfig"; then + if [[ "${BTC_T_ARCH}" == "tilegx" ]]; then + echo ">> [WARN] ${defconfig_target} not available in ${kernel_src}; falling back to defconfig." + f_exec_log "make ARCH=${kernel_arch} defconfig" "kernel-bin-defconfig" + else + echo ">> [ERROR] ${defconfig_target} failed for ${kernel_src}." + return 1 + fi + fi + # Inject Custom Enterprise Swarm Labels & Architecture Parameters sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-dcosnet-${SYS_LABEL}\"/" .config @@ -1146,8 +1360,15 @@ function f_kernel_binary() { echo "CONFIG_KALLSYMS=n" >> .config echo "CONFIG_DEBUG_FS=n" >> .config - # Cross-compile kernel for non-x86_64 targets - local kernel_make_vars="ARCH=${BTC_T_ARCH} CROSS_COMPILE=${TARGET}-" + # Cross-compile kernel using the canonical ARCH= name and the + # Stage 2 cross-compiler (now on PATH after f_gcc_p2()). + # + # LD_LIBRARY_PATH was already set up earlier in this function (before + # the defconfig call) so the loader finds Stage 2's libstdc++.so / + # libgcc_s.so first. See the comment near the top of this function + # for the full rationale. + + local kernel_make_vars="ARCH=${kernel_arch} CROSS_COMPILE=${TARGET}-" f_exec_log "make ${kernel_make_vars} olddefconfig" "kernel-bin-config-merge" f_exec_log "make ${v_threads} ${kernel_make_vars} LOCALVERSION=-dcosnet-${SYS_LABEL}" "kernel-bin-make" @@ -1197,7 +1418,7 @@ function f_package() { cat > "${manifest}" << MANIFEST_EOF { - "btc_version": "0.4.0", + "btc_version": "${BTC_VERSION}", "mode": "${mode_label}", "cross_mode": ${CROSS_MODE}, "sys_label": "${SYS_LABEL}", @@ -1279,7 +1500,7 @@ function f_main() { exit 0 ;; --help|-h) - echo "BTC-0.4.0.sh - Cross-Compilation Build Tool Chain" + echo "BTC-${BTC_VERSION} - Cross-Compilation Build Tool Chain" echo "" echo "Usage: BTC.sh [TARGET_ID | --native | --list | --list-json]" echo " BTC.sh --tpm-seal [TARGET_ID]" diff --git a/NOTES.md b/NOTES.md old mode 100755 new mode 100644 index 3f7e04c..63bc512 --- a/NOTES.md +++ b/NOTES.md @@ -23,8 +23,8 @@ produce target binaries — every target has its own dedicated cross-toolchain. ### Table-Driven Target Registry -All 19 targets are defined in a single associative array -(`BTC_TARGETS[]`). Each entry specifies ten fields in pipe-delimited format: +All 22 targets are defined in a single associative array +(`BTC_TARGETS[]`). Each entry specifies eleven fields in pipe-delimited format: arch|multilib_arch|march|ISA|abi|libc|endian|family|description|min_kernel|gcc_extra @@ -44,11 +44,16 @@ toolchain is packaged into its golden image tarball. Every binary produced by a BTC-built toolchain carries two immutable identifiers: -1. **ELF `.note.BTC` section** — note name "BTC", note type 0xB7C (vendor), - containing a pipe-delimited string with org, version, target, march, ISA, - and a bare hex SHA-256 hash of the source tarball. -2. **Extended attributes (xattr)** — the same stamp data is written to - `user.btc.stamp` on the binary file. +1. **ELF `.note.BTC` section** — note name "DCOSNET", note type 0xB7C + (vendor-specific), containing a pipe-delimited string with org, kernel + version, target ID, march, ISA, sys_label, build stage, signature tier, + and the active signature token. +2. **Extended attributes (xattr)** — four attributes written to the + binary file: + - `user.btc.identity` — `BTC---` + - `user.btc.hash` — SHA-256 of the stamped binary + - `user.btc.sig.tier` — `poly` | `tpm` | `cluster` + - `user.btc.sig.token` — the active signature token These stamps allow any binary to be traced back to the exact build environment, toolchain version, and source tree that produced it. @@ -77,19 +82,29 @@ determines the optimization flags passed to GCC: | TILE | tilegx | (arch set per target) | The SSE4_2 tier exists because Intel Atom and AMD APU low-power cores lack -AVX support. GCC is configured with `--with-arch=` and -`--with-cpu=` in both Stage 1 and Stage 2 to ensure the cross-compiler -defaults to the correct target microarchitecture. +AVX support. GCC is configured with `--with-arch=` in both Stage 1 +and Stage 2 to ensure the cross-compiler defaults to the correct target +microarchitecture. `--with-cpu=` is added for x86_64 targets only — +ARM, MIPS, and TILE backends reject `--with-cpu=` (they want a CPU +name like `cortex-a9`, not an architecture name like `armv7-a`) and instead +carry their per-target `--with-*` hints via `BTC_T_GCC_EXTRA`. ## LFS Base Standards -BTC.sh follows Linux From Scratch 13.0 stable (released 2024-09-01): +BTC.sh follows Linux From Scratch 13.0 stable (released 2024-09-01), bumped +forward to the latest point releases actually downloadable from upstream +mirrors as of v0.4.2: -- Binutils 2.46 -- GCC 14.2.0 -- Glibc 2.41 -- musl 1.2.5 -- Linux kernel headers (matched to target `min_kernel`) +- Binutils 2.46.1 (LFS 13.0 ships 2.46) +- GCC 15.3.0 (LFS 13.0 ships 14.2.0; bumped to 15.3.0 for znver4 / + sierraforest march support and GCC 15 stricter + const-correctness — requires `-Wno-error` for + libxcrypt 4.5.2, which `f_libxcrypt()` already passes) +- Glibc 2.43 (LFS 13.0 ships 2.41) +- musl 1.2.6 (LFS 13.0 ships 1.2.5) +- Linux 7.1.7 (latest 7.1.x point release; LFS 13.0 ships 6.10) +- Tile-Gx override: Linux 5.4.302 LTS + GCC 10.3.0 (mainline dropped + tile in Linux 5.9 / GCC 12) ## C Library Selection Rationale @@ -115,9 +130,12 @@ attribution purposes and are not directly incorporated into the script: ## Source Cache Policy -BTC.sh caches all downloaded source tarballs locally to avoid placing -unnecessary load on upstream hosting infrastructure. Automated bulk downloads -should be rate-limited and sources retained after initial fetch. +BTC.sh caches all downloaded source tarballs locally under `/opt/BTC/src/` +to avoid placing unnecessary load on upstream hosting infrastructure. +Automated bulk downloads should be rate-limited and sources retained after +initial fetch. The cache is reused across builds; the integrity check in +`_archive_sane()` automatically discards and re-fetches corrupted archives +on the next run. ## License diff --git a/btc-quickstart.md b/btc-quickstart.md old mode 100755 new mode 100644 index 62a7e61..7a649fc --- a/btc-quickstart.md +++ b/btc-quickstart.md @@ -1,6 +1,6 @@ -# BTC Quickstart — Version 0.4.1 +# BTC Quickstart — Version 0.4.2 -A guide to building your first cross-toolchain with BTC.sh . +A guide to building your first cross-toolchain with BTC.sh. ## 1. Prerequisites @@ -19,17 +19,17 @@ executes in four phases: available RAM to prevent LTO thrashing. The target registry is loaded. 2. **Setup** — A volatile cleanroom (ramfs) is provisioned at the configured mount point. Source tarballs are verified against their SHA-256 checksums. -3. **STOP USING THIS WORD >>>Build** — Core components are built sequentially: +3. **Build** — Core components are built sequentially: Binutils → Kernel Headers → GCC Stage 1 → C Library (glibc or musl) → GCC Stage 2 → Kernel. - Both GCC stages are configured with `--with-arch=` and `--with-cpu=` - to default to the target microarchitecture. + Both GCC stages are configured with `--with-arch=` (and `--with-cpu=` + for x86_64 targets only) to default to the target microarchitecture. 4. **Package** — The resulting cross-toolchain is compressed into a golden image tarball. A manifest JSON sidecar and forensic ELF stamp are applied. ## 3. Build a Cross-Toolchain ```bash -# List all 19 available targets +# List all 22 available targets sudo ./BTC.sh --list # Build a cross-toolchain for AMD Zen3 (Ryzen 5000 / EPYC Milan) @@ -48,18 +48,20 @@ sudo ./BTC.sh --native ## 4. Verify the Golden Image After a successful build, the golden image tarball and its manifest are written -to `/opt/BTC/releases/`: +directly to `/opt/BTC/` (the manifest uses the `${SYS_LABEL}-manifest.json` +naming convention; the golden image uses `${SYS_LABEL}-toolchain-golden.tar.xz`): ```bash # List available golden images -ls -la /opt/BTC/releases/ +ls -la /opt/BTC/*-toolchain-golden.tar.xz /opt/BTC/*-manifest.json # Inspect the manifest -cat /opt/BTC/releases/DCOSNET-amd-znver3-AVX2-CROSS-toolchain-manifest.json +cat /opt/BTC/DCOSNET-AMD-ZNVER3-AVX2-CROSS-manifest.json ``` The manifest contains structured metadata: target ID, architecture, C library, -microarchitecture, ISA tier, cross-compiler triple, and build timestamps. +microarchitecture, ISA tier, cross-compiler triple, signature tier, and build +configuration flags. ## 5. Forensic Stamp Verification @@ -68,10 +70,10 @@ section. Verify it: ```bash # Read the ELF note -readelf -n /path/to/binary | grep -A5 BTC +readelf -n /path/to/binary | grep -A5 DCOSNET -# Read the xattr stamp -getfattr -d user.btc.stamp /path/to/binary +# Read the four xattr stamps +getfattr -d user.btc.identity,user.btc.hash,user.btc.sig.tier,user.btc.sig.token /path/to/binary ``` ## 6. Integration with Sorcery-Go and Fester @@ -84,7 +86,7 @@ LDFLAGS), and verifies stamps on build outputs. ## 7. Source Cache -BTC.sh caches downloaded source tarballs in `/opt/BTC/sources/`. If a tarball -is already present and its checksum matches, it is not re-downloaded. Keep the -cache directory intact between builds to avoid unnecessary load on upstream -mirrors. \ No newline at end of file +BTC.sh caches downloaded source tarballs in `/opt/BTC/src/`. If a tarball +is already present and its integrity check passes, it is not re-downloaded. +Keep the cache directory intact between builds to avoid unnecessary load on +upstream mirrors. \ No newline at end of file diff --git a/cleanup.sh b/cleanup.sh old mode 100755 new mode 100644 index 6fc6ea5..fb6772c --- a/cleanup.sh +++ b/cleanup.sh @@ -1,11 +1,124 @@ #!/bin/bash -# On target: -cd /opt/BTC -rm -rf BTC-0.4.1 -umount -l /usr/src 2>/dev/null || true -rm -rf /usr/src/DCOSNET-HASWELL-AVX2-LTO-cleanroom -rm -rf logs/DCOSNET-HASWELL-AVX2-LTO +# cleanup.sh — post-build janitor for BTC.sh +# +# Removes the volatile cleanroom, unmounts the ramfs, and re-stages a fresh +# BTC.sh copy under /opt/BTC/BTC-/ ready to be invoked. +# +# Usage: +# ./cleanup.sh # uses BTC_VERSION from BTC.sh auto-detect +# ./cleanup.sh # e.g. ./cleanup.sh haswell +# ./cleanup.sh # e.g. ./cleanup.sh haswell 0.4.2 +# +# Defaults: +# target_id = haswell (matches the original cleanup.sh behavior) +# version = read from BTC.sh (falls back to 0.4.2) +# +# NOTE: this script must be run on the target machine AFTER the BTC.sh build +# host has produced the golden image. It does NOT rebuild anything — it +# just clears the working state and re-stages the next BTC.sh copy. + +set -euo pipefail + +BTC_ARCHIVE="/opt/BTC" +SOURCES_ACTIVE="/usr/src" + +# --- Resolve target_id argument ----------------------------------------- +# The original cleanup.sh hardcoded DCOSNET-HASWELL-AVX2-LTO. We default +# to the same target so behavior is unchanged for users who run the script +# with no arguments, but expose the full SYS_LABEL derivation so any of the +# 22 registered targets can be cleaned up by passing its target_id. +TARGET_ID="${1:-haswell}" + +# --- Resolve BTC_VERSION ------------------------------------------------- +# Prefer an explicit second arg, otherwise parse it out of the BTC.sh +# sitting next to this script so the cleanup tracks the deployed version +# automatically. Falls back to 0.4.2 if neither is available. +BTC_VERSION="${2:-}" +if [[ -z "${BTC_VERSION}" ]]; then + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + BTC_SH="${SCRIPT_DIR}/BTC.sh" + if [[ -f "${BTC_SH}" ]]; then + BTC_VERSION="$(grep -m1 -E '^readonly BTC_VERSION=' "${BTC_SH}" \ + | sed -E 's/^readonly BTC_VERSION="([^"]+)".*$/\1/' || true)" + fi +fi +BTC_VERSION="${BTC_VERSION:-0.4.2}" + +# --- Derive SYS_LABEL the same way BTC.sh does --------------------------- +# For native x86_64 builds BTC.sh produces SYS_LABEL=DCOSNET---LTO. +# For cross builds it produces SYS_LABEL=DCOSNET----CROSS. +# The cleanup needs to match the build's actual label, so we use the same +# uppercase + family/ISA tags. ISA is read from BTC_TARGETS[] if BTC.sh +# is reachable; otherwise we conservatively probe by family. +FAMILY="" +case "${TARGET_ID}" in + # AVX512 Intel HEDT/Server (must come before the AVX2 catch-all) + skylake-x|skylake-server) + FAMILY="intel"; ISA_TAG="AVX512" ;; + # AVX2 Intel HEDT/Server + haswell|haswell-ep|broadwell|broadwell-ep|skylake) + FAMILY="intel"; ISA_TAG="AVX2" ;; + # AVX512 AMD + znver4) + FAMILY="amd"; ISA_TAG="AVX512" ;; + # AVX2 AMD + znver1|znver2|znver3) + FAMILY="amd"; ISA_TAG="AVX2" ;; + apu-zn1|apu-zn2|apu-zn3|apu-zn4) + FAMILY="amd-apu"; ISA_TAG="AVX2" ;; + # SSE4_2 Intel Atom + atom-silvermont|atom-goldmont|atom-tremont|atom-sierraforest) + FAMILY="atom"; ISA_TAG="SSE4_2" ;; + # Embedded + mipselr2) + FAMILY="mips"; ISA_TAG="MIPS32" ;; + armv7) + FAMILY="arm"; ISA_TAG="NEON" ;; + tilegx) + FAMILY="tile"; ISA_TAG="TILE" ;; + *) + echo ">> [ERROR] Unknown target_id: ${TARGET_ID}" + echo ">> Run 'BTC.sh --list' to see registered targets." + exit 1 + ;; +esac + +# Mirror BTC.sh's native-vs-cross SYS_LABEL rule: +# native -> DCOSNET---LTO +# cross -> DCOSNET----CROSS +# x86_64 targets are typically run native; everything else is cross. +if [[ "${FAMILY}" == "intel" || "${FAMILY}" == "amd" || "${FAMILY}" == "amd-apu" || "${FAMILY}" == "atom" ]]; then + SYS_LABEL="DCOSNET-${TARGET_ID^^}-${ISA_TAG}-LTO" +else + SYS_LABEL="DCOSNET-${FAMILY^^}-${TARGET_ID^^}-${ISA_TAG}-CROSS" +fi + +echo ">> [CLEANUP] target_id=${TARGET_ID} version=${BTC_VERSION} sys_label=${SYS_LABEL}" + +# --- Tear down the previous build state ---------------------------------- +cd "${BTC_ARCHIVE}" +rm -rf "BTC-${BTC_VERSION}" + +# Lazy-unmount the ramfs cleanroom (in case a previous run crashed mid-build). +umount -l "${SOURCES_ACTIVE}" 2>/dev/null || true + +# Remove the target-specific cleanroom directory and its logs. +rm -rf "${SOURCES_ACTIVE}/${SYS_LABEL}-cleanroom" +rm -rf "${BTC_ARCHIVE}/logs/${SYS_LABEL}" + +# --- Re-stage the next BTC.sh copy --------------------------------------- # Upload the fixed BTC.sh from download/, then: -mkdir -p BTC-0.4.1 -cp BTC.sh BTC-0.4.1/ -cd BTC-0.4.1 && chmod +x BTC.sh && echo "btc ready in /opt/BTC/BTC-0.4.1/" #./BTC.sh \ No newline at end of file +mkdir -p "BTC-${BTC_VERSION}" +if [[ -f "${BTC_ARCHIVE}/BTC.sh" ]]; then + cp "${BTC_ARCHIVE}/BTC.sh" "BTC-${BTC_VERSION}/" +elif [[ -f "./BTC.sh" ]]; then + cp "./BTC.sh" "BTC-${BTC_VERSION}/" +else + echo ">> [WARN] No BTC.sh found to stage under BTC-${BTC_VERSION}/." + echo ">> Drop BTC.sh into ${BTC_ARCHIVE}/ or ${PWD}/ and re-run." +fi + +cd "BTC-${BTC_VERSION}" && chmod +x BTC.sh +echo ">> [READY] btc staged at ${BTC_ARCHIVE}/BTC-${BTC_VERSION}/BTC.sh" +echo ">> Target label cleaned: ${SYS_LABEL}" +echo ">> Invoke with: ./BTC.sh ${TARGET_ID}"