sorcery-go/scripts/smgl-getting-started.sh

507 lines
22 KiB
Bash
Executable File

#!/bin/bash
# smgl-getting-started.sh
# =============================================================================
# Resurrect an old Source Mage chroot tarball (0.62-11 / 0.63 test branch)
# and pull it into the modern era — then drop in sorcery-go as the new engine.
#
# NOTE: sorcery-go is developed by dcos.net and is NOT affiliated with
# Source Mage GNU/Linux or sourcemage.org. This script downloads resources
# from sourcemage.org (a separate, independent project) for convenience.
#
# This script encodes the 8-phase staging pipeline from the Source Mage
# chroot path document. It is split into HOST-side subcommands (run from
# your modern host OS) and CHROOT-side subcommands (run after you `chroot`
# into the staged environment).
#
# HOST subcommands (run as root on your modern host):
#
# extract <tarball> <mountpoint>
# Extract the SMGL rootfs tarball to <mountpoint>.
# mount <mountpoint>
# Bind-mount /dev /proc /sys /etc/resolv.conf into the chroot.
# inject-sorcery <mountpoint> [sorcery-tarball]
# Download (or use the provided) modern sorcery engine tarball
# and overlay it into the chroot's /usr and /etc.
# inject-kernel <mountpoint> <bzImage> [modules-dir]
# Copy a modern host-built kernel + modules into the chroot.
# Strips the old /boot/vmlinuz* and /lib/modules/* first.
# inject-sorcery-go <mountpoint> [sorcery-go-binary]
# Drop the sorcery-go binary + state dirs into the chroot so
# it coexists with legacy /usr/sbin/sorcery.
# fix-fstab <mountpoint> <root-uuid> [<boot-uuid>]
# Rewrite /etc/fstab to use UUID= identifiers.
# enter <mountpoint>
# chroot into the staged environment with a login shell.
# unmount <mountpoint>
# Unmount the API filesystems (safe to run multiple times).
#
# CHROOT subcommands (run AFTER `enter`, from inside the chroot):
#
# chroot-purge-grub1
# Remove /boot/grub/menu.lst and dispel the legacy grub spell.
# chroot-cast-grub2
# cast grub2 (skips gracefully if the toolchain is too old).
# chroot-scribe-test
# Drop the dead stable grimoire and anchor scribe to the
# live test branch from git://download.sourcemage.org/smgl/grimoire.git
# chroot-stepupgrade
# Run the step-upgrade ladder: make → binutils → gcc → glibc.
# This is the most fragile phase — read the docs first.
# chroot-init-sorcery-go
# Initialise the sorcery-go state DB inside the chroot and
# index the test grimoire.
#
# USAGE EXAMPLES:
#
# # Full host-side prep
# sudo ./scripts/smgl-getting-started.sh extract \
# ~/Downloads/source-mage-x86_64-0.62-11.tar.xz /mnt/AI/sourcemage_root
# sudo ./scripts/smgl-getting-started.sh mount /mnt/AI/sourcemage_root
# sudo ./scripts/smgl-getting-started.sh inject-sorcery /mnt/AI/sourcemage_root
# sudo ./scripts/smgl-getting-started.sh inject-kernel /mnt/AI/sourcemage_root \
# /usr/src/linux/arch/x86/boot/bzImage /lib/modules/6.x-custom
# sudo ./scripts/smgl-getting-started.sh inject-sorcery-go /mnt/AI/sourcemage_root
# sudo ./scripts/smgl-getting-started.sh enter /mnt/AI/sourcemage_root
#
# # Inside the chroot now:
# sourcemage-getting-started.sh chroot-scribe-test
# sourcemage-getting-started.sh chroot-stepupgrade
# sourcemage-getting-started.sh chroot-init-sorcery-go
# exit
#
# # Back on host:
# sudo ./scripts/smgl-getting-started.sh unmount /mnt/AI/sourcemage_root
# =============================================================================
set -euo pipefail
# --- pretty output ---
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
phase() { echo -e "\n${BLUE}━━━ Phase: $1 ━━━${NC}"; }
ok() { echo -e "${GREEN}${NC} $1"; }
warn() { echo -e "${YELLOW}${NC} $1"; }
err() { echo -e "${RED}${NC} $1" >&2; }
die() { err "$1"; exit 1; }
# Resolve the project root so we can find the build/sorcery binary and the
# manifests directory regardless of where the script is invoked from.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
usage() {
# Print every comment line from the script header, skipping the
# === divider lines. Stop at the first non-comment line.
awk '/^#!/{next} /^# ===/{next} /^# /{sub(/^# ?/,""); print; next} {exit}' "$0"
echo ""
echo "Subcommands:"
echo " HOST: extract | mount | unmount | inject-sorcery | inject-kernel"
echo " inject-sorcery-go | fix-fstab | enter"
echo " CHROOT: chroot-purge-grub1 | chroot-cast-grub2 | chroot-scribe-test"
echo " chroot-stepupgrade | chroot-init-sorcery-go"
echo ""
echo "Run with --help and a subcommand for details, or see"
echo "docs/GETTING_STARTED_SMGL_CHROOT.md for the full walkthrough."
exit "${1:-1}"
}
require_root() {
if [ "$(id -u)" -ne 0 ]; then
die "this subcommand requires root. Re-run with: sudo $0 $*"
fi
}
# =============================================================================
# HOST-SIDE SUBCOMMANDS
# =============================================================================
cmd_extract() {
[ $# -lt 2 ] && die "usage: extract <tarball> <mountpoint>"
local tarball="$1" mountpoint="$2"
require_root extract
[ -f "$tarball" ] || die "tarball not found: $tarball"
mkdir -p "$mountpoint"
phase "1.1 — Extract Source Mage RootFS"
echo " tarball: $tarball"
echo " mountpoint: $mountpoint"
case "$tarball" in
*.tar.xz) tar -xJf "$tarball" -C "$mountpoint" ;;
*.tar.bz2) tar -xjf "$tarball" -C "$mountpoint" ;;
*.tar.gz) tar -xzf "$tarball" -C "$mountpoint" ;;
*.tar) tar -xf "$tarball" -C "$mountpoint" ;;
*) die "unrecognised tarball extension: $tarball" ;;
esac
ok "RootFS extracted to $mountpoint"
}
cmd_mount() {
[ $# -lt 1 ] && die "usage: mount <mountpoint>"
local mp="$1"
require_root mount
[ -d "$mp" ] || die "mountpoint does not exist: $mp"
phase "1.2 — Mount API filesystems into chroot"
# Use --rbind so submounts (dev/pts, dev/shm) come along automatically.
mount --rbind /dev "$mp/dev" 2>/dev/null || warn "dev already mounted"
mount --rbind /proc "$mp/proc" 2>/dev/null || warn "proc already mounted"
mount --rbind /sys "$mp/sys" 2>/dev/null || warn "sys already mounted"
# resolv.conf must be a regular file copy, not a bind, so the chroot's
# network config survives even if the host's resolv.conf is later rotated.
cp /etc/resolv.conf "$mp/etc/resolv.conf" 2>/dev/null || \
warn "could not copy /etc/resolv.conf (host may use systemd-resolved)"
ok "API filesystems mounted. Ready to chroot with:"
echo " sudo chroot $mp /bin/bash --login"
echo " or:"
echo " sudo $0 enter $mp"
}
cmd_unmount() {
[ $# -lt 1 ] && die "usage: unmount <mountpoint>"
local mp="$1"
require_root unmount
phase "Cleanup — Unmount API filesystems"
# Unmount in reverse order, recursively, lazy as a fallback.
for sub in dev/pts dev/shm dev proc sys; do
umount -f "$mp/$sub" 2>/dev/null || true
done
umount -R "$mp" 2>/dev/null || umount -l "$mp" 2>/dev/null || true
ok "Unmounted. The rootfs at $mp is now safe to tar / dd / delete."
}
cmd_inject_sorcery() {
[ $# -lt 1 ] && die "usage: inject-sorcery <mountpoint> [sorcery-tarball]"
local mp="$1" sorcery_tarball="${2:-}"
require_root inject-sorcery
phase "5 — Inject modern Sorcery engine"
local tmpdir; tmpdir="$(mktemp -d)"
trap "rm -rf $tmpdir" EXIT
if [ -z "$sorcery_tarball" ]; then
warn "no sorcery tarball provided — downloading from sourcemage.org"
sorcery_tarball="$tmpdir/sorcery-stable.tar.bz2"
# The official stable tarball — we overlay it onto the chroot so the
# /usr/sbin/* and /etc/sorcery/* scripts get refreshed.
wget -q -O "$sorcery_tarball" \
"https://sourcemage.org/codex/sorcery-stable.tar.bz2" || \
die "download failed. Pass a local tarball: inject-sorcery <mp> <tarball>"
fi
[ -f "$sorcery_tarball" ] || die "sorcery tarball not found: $sorcery_tarball"
echo " extracting $sorcery_tarball..."
tar -xjf "$sorcery_tarball" -C "$tmpdir"
# The sorcery tarball layout mirrors the FHS — usr/* etc/sorcery/*.
# Overlay it onto the chroot (don't delete existing files; just replace).
if [ -d "$tmpdir/usr" ]; then
cp -a "$tmpdir/usr/." "$mp/usr/"
ok "modern /usr/sbin/* scripts injected"
fi
if [ -d "$tmpdir/etc/sorcery" ]; then
mkdir -p "$mp/etc/sorcery"
cp -a "$tmpdir/etc/sorcery/." "$mp/etc/sorcery/"
ok "modern /etc/sorcery config injected"
fi
warn "if the chroot's old sorcery config conflicts, review $mp/etc/sorcery/ by hand"
}
cmd_inject_kernel() {
[ $# -lt 2 ] && die "usage: inject-kernel <mountpoint> <bzImage> [modules-dir]"
local mp="$1" bzImage="$2" modules_dir="${3:-}"
require_root inject-kernel
[ -f "$bzImage" ] || die "bzImage not found: $bzImage"
phase "3 — Strip old kernel + inject modern host-built kernel"
# 3.1 — purge the old /boot noise and /lib/modules/*.
echo " purging /boot/vmlinuz* /boot/initrd* /lib/modules/* ..."
rm -f "$mp"/boot/vmlinuz* "$mp"/boot/initrd* 2>/dev/null || true
rm -rf "$mp"/lib/modules/* 2>/dev/null || true
# 3.2 — copy in the modern kernel binary.
local kver; kver="$(basename "$bzImage" | sed 's/^vmlinuz-//; s^-^_^g')"
# If the bzImage basename already encodes a version (e.g. vmlinuz-6.12.4-custom),
# keep it; otherwise derive one from the file.
local dest_name; dest_name="$(basename "$bzImage")"
case "$dest_name" in
vmlinuz-*|bzImage-*) : ;;
bzImage|vmlinuz) dest_name="vmlinuz-modern" ;;
esac
cp -a "$bzImage" "$mp/boot/$dest_name"
ok "kernel image installed: /boot/$dest_name"
# 3.3 — optionally copy the matching module tree.
if [ -n "$modules_dir" ] && [ -d "$modules_dir" ]; then
mkdir -p "$mp/lib/modules"
cp -a "$modules_dir/." "$mp/lib/modules/"
ok "module tree installed: /lib/modules/$(basename "$modules_dir")"
else
warn "no modules-dir provided — kernel will boot but won't load any modules."
warn " copy them with: sudo cp -a /lib/modules/<ver> $mp/lib/modules/"
fi
# 3.4 — touch /etc/hostname so the chroot has a distinct identity.
if [ ! -s "$mp/etc/hostname" ]; then
echo "smgl-staged" > "$mp/etc/hostname"
ok "default hostname set: smgl-staged (edit $mp/etc/hostname to change)"
fi
}
cmd_inject_sorcery_go() {
[ $# -lt 1 ] && die "usage: inject-sorcery-go <mountpoint> [sorcery-go-binary]"
local mp="$1" binary="${2:-$PROJECT_ROOT/build/sorcery}"
require_root inject-sorcery-go
[ -x "$binary" ] || die "sorcery-go binary not found: $binary (run 'make build' first)"
phase "Drop in sorcery-go (coexists with legacy /usr/sbin/sorcery)"
# Install as sorcery-go so it never overwrites the legacy sorcery binary.
install -d "$mp/usr/local/sbin"
install -m 755 "$binary" "$mp/usr/local/sbin/sorcery-go"
# Create the state directory — separate from /var/lib/sorcery so the
# legacy Bash state stays untouched.
install -d -m 700 "$mp/var/lib/sorcery-go/state"
install -d -m 700 "$mp/var/lib/sorcery-go/tomb/epitaphs"
install -d -m 700 "$mp/var/lib/sorcery-go/tomb/blobs"
install -d -m 755 "$mp/var/lib/sorcery-go/build"
install -d -m 755 "$mp/var/lib/sorcery-go/log"
install -d -m 755 "$mp/var/spool/sorcery-go"
# Apply capabilities if the host supports setcap.
# Include CAP_BPF for eBPF program loading.
if command -v setcap >/dev/null 2>&1; then
setcap 'cap_sys_admin,cap_chown,cap_dac_override,cap_bpf+ep' \
"$mp/usr/local/sbin/sorcery-go" 2>/dev/null || \
warn "setcap failed — sorcery-go will need to run as root"
fi
# Create eBPF maps directory for pinned maps.
install -d -m 755 "$mp/var/lib/sorcery-go/ebpf/maps" 2>/dev/null || true
ok "sorcery-go dropped in at $mp/usr/local/sbin/sorcery-go"
echo " state root: $mp/var/lib/sorcery-go/"
echo " spool: $mp/var/spool/sorcery-go/"
echo " eBPF maps: $mp/var/lib/sorcery-go/ebpf/maps/"
echo " runtime: auto-detect (lxc > podman > firecracker > baremetal)"
warn "do NOT run sorcery-go init until AFTER the step-upgrade ladder completes"
warn " (the chroot-init-sorcery-go subcommand runs it at the right time)"
}
cmd_fix_fstab() {
[ $# -lt 2 ] && die "usage: fix-fstab <mountpoint> <root-uuid> [<boot-uuid>]"
local mp="$1" root_uuid="$2" boot_uuid="${3:-}"
phase "4 — Rewrite /etc/fstab with persistent UUID identifiers"
local fstab="$mp/etc/fstab"
[ -f "$fstab" ] || { warn "$fstab missing — creating fresh"; touch "$fstab"; }
# Back up the original.
cp -a "$fstab" "$fstab.pre-sorcery-go.bak"
{
echo "# /etc/fstab — rewritten by smgl-getting-started.sh fix-fstab"
echo "# original backed up at $fstab.pre-sorcery-go.bak"
echo "UUID=$root_uuid / ext4 noatime,defaults 0 1"
if [ -n "$boot_uuid" ]; then
echo "UUID=$boot_uuid /boot vfat defaults 0 2"
fi
echo "proc /proc proc defaults 0 0"
echo "sysfs /sys sysfs defaults 0 0"
echo "devpts /dev/pts devpts gid=5,mode=620 0 0"
echo "tmpfs /tmp tmpfs defaults 0 0"
} > "$fstab"
ok "fstab rewritten with UUIDs. Review with: cat $fstab"
}
cmd_enter() {
[ $# -lt 1 ] && die "usage: enter <mountpoint>"
local mp="$1"
require_root enter
[ -d "$mp" ] || die "mountpoint does not exist: $mp"
phase "Entering chroot (Ctrl+D or 'exit' to leave)"
# Copy this script into the chroot so the chroot-* subcommands work inside.
if [ ! -x "$mp/usr/local/sbin/smgl-getting-started.sh" ]; then
install -d "$mp/usr/local/sbin"
install -m 755 "$0" "$mp/usr/local/sbin/smgl-getting-started.sh"
fi
exec chroot "$mp" /bin/bash --login
}
# =============================================================================
# CHROOT-SIDE SUBCOMMANDS
# (run AFTER `enter` — these expect to be inside the staged environment)
# =============================================================================
cmd_chroot_purge_grub1() {
phase "2.1 — Purge GRUB Legacy"
if [ -f /boot/grub/menu.lst ]; then
rm -f /boot/grub/menu.lst
ok "removed /boot/grub/menu.lst"
else
warn "/boot/grub/menu.lst not present — already purged"
fi
if command -v dispel >/dev/null 2>&1; then
echo " dispelling legacy grub spell..."
dispel grub || warn "dispel grub failed — old spell may not be indexed"
else
warn "dispel not found in PATH — sorcery not yet injected? Run inject-sorcery first"
fi
}
cmd_chroot_cast_grub2() {
phase "2.2 — Cast GRUB 2"
if ! command -v cast >/dev/null 2>&1; then
err "cast not found — run inject-sorcery from the host first"
err "you can also skip this and run grub-install from the modern host later:"
err " grub-install --boot-directory=$mp/boot /dev/sdX"
return 1
fi
echo " casting grub2 (this may fail if the toolchain is too ancient — see Phase 8)..."
cast grub2 || {
warn "cast grub2 failed — this is expected on a 2017-era toolchain."
warn "skip it for now and run grub-install from the modern host after the step-upgrade."
}
}
cmd_chroot_scribe_test() {
phase "7 — Cut over from dead stable grimoire to live test branch"
if ! command -v scribe >/dev/null 2>&1; then
die "scribe not found — run inject-sorcery from the host first"
fi
echo " current grimoire index:"
scribe index || true
echo " dropping dead stable codex..."
scribe remove stable || warn "stable not present (already removed?)"
echo " anchoring scribe to live test branch..."
# Try the canonical git URL first; fall back to https if git:// is firewalled.
if scribe add test from git://download.sourcemage.org/smgl/grimoire.git; then
ok "scribe anchored to test branch (git://)"
elif scribe add test from https://download.sourcemage.org/smgl/grimoire.git; then
ok "scribe anchored to test branch (https fallback)"
else
err "scribe add test failed — try cloning manually:"
err " git clone https://download.sourcemage.org/smgl/grimoire.git /var/lib/sorcery/codex/test"
return 1
fi
scribe index
ok "test grimoire indexed. Modern spells (GCC 15+, 6.x kernel configs) now visible."
}
cmd_chroot_stepupgrade() {
phase "8 — Step-upgrade the toolchain ladder (make → binutils → gcc → glibc)"
if ! command -v cast >/dev/null 2>&1; then
die "cast not found — run inject-sorcery from the host first"
fi
# Phase 6 — set conservative CFLAGS so the ancient compiler doesn't choke.
phase "6 — Set conservative optimisation (CFLAGS=-O2 -march=native)"
if command -v sorcery >/dev/null 2>&1; then
# Non-interactive: write the optimisation directly to the sorcery config.
# The interactive `sorcery` menu is the canonical way; this is the
# scriptable equivalent.
cat >> /etc/sorcery/config <<'EOF'
# --- added by smgl-getting-started.sh stepupgrade ---
OPTIMIZATION_FLAGS="-O2 -march=native"
GCC_SPIKE_TEST=n
# remove experimental flags for the first wave
JOBS_PER_HOST=$(nproc)
EOF
ok "conservative CFLAGS written to /etc/sorcery/config"
track_root / 2>/dev/null || true
else
warn "sorcery menu not available — set OPTIMIZATION_FLAGS by hand"
fi
echo ""
echo " This is the most fragile phase of the resurrection."
echo " If any step segfaults, see docs/GETTING_STARTED_SMGL_CHROOT.md"
echo " for the 'inject host-built binary' escape hatch."
echo ""
read -rp " Continue with the step-upgrade ladder? [y/N] " confirm
[ "$confirm" = "y" ] || { warn "aborted — re-run when ready"; return 1; }
phase "8.1 — cast make"
cast make || die "cast make failed — toolchain too ancient. Use the host-binary escape hatch."
phase "8.2 — cast binutils"
cast binutils || die "cast binutils failed"
phase "8.3 — cast gcc (intermediate stepping stone)"
cast gcc || die "cast gcc failed — try an explicit intermediate version (GCC 9 or 10)"
phase "8.4 — cast glibc (the cutover)"
cast glibc || die "cast glibc failed — the runtime is now in an inconsistent state. See docs."
ok "Toolchain step-upgrade complete!"
ok "Modern syscall wrappers (statx, clone3) are now bound to the chroot."
echo ""
echo " Next: run 'smgl-getting-started.sh chroot-init-sorcery-go' to bring"
echo " the new Go engine online alongside the legacy Bash sorcery."
}
cmd_chroot_init_sorcery_go() {
phase "Drop-in: initialise sorcery-go state DB inside the chroot"
if [ ! -x /usr/local/sbin/sorcery-go ]; then
die "sorcery-go binary not found at /usr/local/sbin/sorcery-go"
err "run inject-sorcery-go from the host first"
return 1
fi
# The grimoire path inside the chroot is whatever scribe just indexed.
local grimoire_path="/var/lib/sorcery/codex/test"
[ -d "$grimoire_path" ] || grimoire_path="/var/lib/sorcery/codex/grimoire"
[ -d "$grimoire_path" ] || {
warn "no grimoire found at /var/lib/sorcery/codex/{test,grimoire}"
warn "sorcery-go will still init — index an empty grimoire for now"
grimoire_path=""
}
if [ -n "$grimoire_path" ]; then
SORCERY_GO_GRIMOIRE="$grimoire_path" \
SORCERY_GO_ROOT=/var/lib/sorcery-go \
SORCERY_GO_SPOOL=/var/spool/sorcery-go \
SORCERY_GO_PGP_KEYRING="" \
/usr/local/sbin/sorcery-go init --force
else
SORCERY_GO_ROOT=/var/lib/sorcery-go \
SORCERY_GO_SPOOL=/var/spool/sorcery-go \
SORCERY_GO_PGP_KEYRING="" \
/usr/local/sbin/sorcery-go init
fi
ok "sorcery-go is live inside the chroot!"
echo ""
echo " Try your first Go-powered cast:"
echo " sorcery-go gaze depends wget"
echo " sudo sorcery-go cast busybox --static --default"
echo " sorcery-go tomb list"
echo ""
echo " Launch the Coven Mirror WebUI:"
echo " sudo sorcery-go web --port 8080"
}
# =============================================================================
# Dispatch
# =============================================================================
main() {
[ $# -lt 1 ] && usage 1
local cmd="$1"; shift
case "$cmd" in
extract) cmd_extract "$@" ;;
mount) cmd_mount "$@" ;;
unmount) cmd_unmount "$@" ;;
inject-sorcery) cmd_inject_sorcery "$@" ;;
inject-kernel) cmd_inject_kernel "$@" ;;
inject-sorcery-go) cmd_inject_sorcery_go "$@" ;;
fix-fstab) cmd_fix_fstab "$@" ;;
enter) cmd_enter "$@" ;;
chroot-purge-grub1) cmd_chroot_purge_grub1 "$@" ;;
chroot-cast-grub2) cmd_chroot_cast_grub2 "$@" ;;
chroot-scribe-test) cmd_chroot_scribe_test "$@" ;;
chroot-stepupgrade) cmd_chroot_stepupgrade "$@" ;;
chroot-init-sorcery-go) cmd_chroot_init_sorcery_go "$@" ;;
-h|--help|help) usage 0 ;;
*) err "unknown subcommand: $cmd"; usage 1 ;;
esac
}
main "$@"