dcosnet
/
Thicket
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Thicket/scripts/bootstrap_sources.sh

329 lines
14 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# bootstrap_sources — source-anchored dependency build into /mnt/AI.
#
# Every Thicket dependency is mirrored as a git checkout under
# <AI_ROOT>/distfiles/git/ and built into THE venv at
# <AI_ROOT>/runtime/thicket-venv — one environment, always, outside
# the project tree so the checkout stays clean for git and archives.
# Launchers land in <AI_ROOT>/tools/bin and ~/.local/bin.
# Idempotent: re-run to update.
#
# Methods (per package, table below):
# source — pip install from the local git checkout (pure Python)
# wheel — pip install the release wheel; the git checkout is still
# mirrored for audit. Native-heavy builds (faiss, lancedb,
# duckdb, fastembed/onnxruntime, PySide6/Qt) produce the
# same code as the wheel at a fraction of the build cost.
# --force-source builds everything from checkouts.
#
# Repository sync is state-aware:
# absent -> atomic clone (staged in a temp dir, moved into place)
# corrupt -> dir without .git is moved aside, then re-cloned
# dirty -> local modifications are never destroyed; the package
# is skipped with the offending files listed
# moved -> fast-forward to FETCH_HEAD only when HEAD differs;
# unchanged checkouts are NOT reinstalled (the venv is
# probed for the importable module first)
# Step-down: a failed source build falls back to the wheel (logged),
# so one broken upstream never wedges the whole bootstrap.
#
# Usage:
# scripts/bootstrap_sources.sh # asks which projects
# (Enter = all) in a
# terminal; installs all
# when non-interactive
# scripts/bootstrap_sources.sh --only a,b # subset, no prompt
# scripts/bootstrap_sources.sh --force-source # build native too
# scripts/bootstrap_sources.sh --update # git pull + reinstall
#
# Native build deps (auto-installed via pacman when missing, sudo):
# git gcc make cmake rustup postgresql-libs
set -euo pipefail
AI_ROOT="${THICKET_AI_ROOT:-/mnt/AI}"
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
GIT_MIRROR="$AI_ROOT/distfiles/git"
VENV_DIR="$AI_ROOT/runtime/thicket-venv"
BIN_DIR="$AI_ROOT/tools/bin"
LOG_DIR="$AI_ROOT/logs"
LOG="$LOG_DIR/thicket-bootstrap.log"
FORCE_SOURCE=0
UPDATE_ONLY=0
ONLY=""
mode="${1:-}"
case "$mode" in
--force-source) FORCE_SOURCE=1 ;;
--update) UPDATE_ONLY=1 ;;
--only) ONLY="${2:-}"; [ -n "$ONLY" ] || { echo "--only needs a list"; exit 2; } ;;
"") ;;
*) echo "unknown option: $mode"; exit 2 ;;
esac
mkdir -p "$GIT_MIRROR" "$VENV_DIR" "$BIN_DIR" "$LOG_DIR"
exec > >(tee -a "$LOG") 2>&1
log() { printf '\033[1;32m>>>\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m>>> WARN\033[0m %s\n' "$*"; }
# ── name | git url | pip name | extras | method ─────────────────────────
# Pure-python projects install from their checkout; the native-heavy
# set keeps a mirrored checkout but installs the release wheel.
PACKAGES=(
"qdrant-client|https://github.com/qdrant/qdrant-client.git|qdrant-client||source|qdrant_client|vector target: qdrant (service)"
"chromadb|https://github.com/chroma-core/chroma.git|chromadb||source|chromadb|vector target: chroma (embedded)"
"lancedb|https://github.com/lancedb/lancedb.git|lancedb||wheel|lancedb|vector target: lancedb (embedded)"
"faiss|https://github.com/facebookresearch/faiss.git|faiss-cpu||wheel|faiss|vector target: faiss (file index)"
"pymilvus|https://github.com/milvus-io/pymilvus.git|pymilvus|[milvus_lite]|source|pymilvus|vector target: milvus lite"
"weaviate-client|https://github.com/weaviate/weaviate-python-client.git|weaviate-client||source|weaviate|vector target: weaviate (service)"
"psycopg|https://github.com/psycopg/psycopg.git|psycopg|[binary]|wheel|psycopg|vector target: pgvector - postgres driver"
"pgvector|https://github.com/pgvector/pgvector-python.git|pgvector||source|pgvector|vector target: pgvector - python bindings"
"duckdb|https://github.com/duckdb/duckdb.git|duckdb||wheel|duckdb|vector target: duckdb (embedded)"
"sqlite-vec|https://github.com/asg017/sqlite-vec.git|sqlite-vec||source|sqlite_vec|vector target: sqlite-vec (embedded)"
"PyMySQL|https://github.com/PyMySQL/PyMySQL.git|PyMySQL||source|pymysql|vector target: mariadb driver"
"graphify|https://github.com/Graphify-Labs/graphify.git|graphifyy|[ollama]|source|graphify|graph engine: graph.json + HTML report"
"vanna|https://github.com/vanna-ai/vanna.git|vanna|[ollama,postgres,mysql]|source|vanna|ask: natural-language SQL over corpora"
"minio-py|https://github.com/minio/minio-py.git|minio||source|minio|stage: MinIO object archive"
"LightRAG|https://github.com/HKUDS/LightRAG.git|lightrag-hku||source|lightrag|graph engine: merged entity graph"
"ebooklib|https://github.com/aerkalov/ebooklib.git|EbookLib||source|ebooklib|parser: EPUB"
"BeautifulSoup4|https://github.com/wention/BeautifulSoup4.git|beautifulsoup4||source|bs4|parser: HTML inside EPUB"
"pypdf|https://github.com/py-pdf/pypdf.git|pypdf||source|pypdf|parser: PDF"
"python-slugify|https://github.com/un33k/python-slugify.git|python-slugify||source|slugify|vault note filenames"
"fastembed|https://github.com/qdrant/fastembed.git|fastembed||wheel|fastembed|local ONNX embeddings"
"PySide6|https://code.qt.io/pyside/pyside-setup.git|PySide6||wheel|PySide6|the console GUI toolkit"
)
# ── native build tools via pacman (skipped when present) ─────────────
required_tools=(git gcc make)
missing=()
for tool in "${required_tools[@]}"; do
command -v "$tool" >/dev/null 2>&1 || missing+=("$tool")
done
if [ "${#missing[@]}" -gt 0 ]; then
log "installing build tools via pacman: ${missing[*]}"
sudo pacman -S --needed "${missing[@]}"
fi
# ── project selection ─────────────────────────────────────────────────
# Interactive in a terminal; --only or non-TTY stdin bypasses the menu.
select_packages() {
if [ -n "$ONLY" ]; then
return
fi
if [ ! -t 0 ] && [ "${THICKET_BOOTSTRAP_INTERACTIVE:-0}" != "1" ]; then
return # everything, no prompt
fi
local -a names=() roles=()
local row
for row in "${PACKAGES[@]}"; do
names+=("$(cut -d'|' -f1 <<<"$row")")
roles+=("$(cut -d'|' -f7 <<<"$row")")
done
printf 'Thicket projects:\n'
local i
for i in "${!names[@]}"; do
printf ' %2d) %-18s %-34s %s\n' \
"$((i + 1))" "${names[$i]}" \
"$(cut -d'|' -f5 <<<"${PACKAGES[$i]}")" "${roles[$i]}"
done
local answer token from to picked valid
while :; do
read -r -p $'Install which? (numbers/names/ranges, "all", Enter = all): ' answer || return
[ -z "$answer" ] && return
[ "$answer" = "all" ] && return
picked=""
for token in ${answer//,/ }; do
valid=""
if [[ "$token" =~ ^[0-9]+$ ]] && [ "$token" -ge 1 ] \
&& [ "$token" -le "${#names[@]}" ]; then
picked+=",${names[$((token - 1))]}"; valid=1
elif [[ "$token" =~ ^([0-9]+)-([0-9]+)$ ]]; then
from="${BASH_REMATCH[1]}"; to="${BASH_REMATCH[2]}"
if [ "$from" -ge 1 ] && [ "$to" -le "${#names[@]}" ] \
&& [ "$from" -le "$to" ]; then
for ((i = from; i <= to; i++)); do
picked+=",${names[$((i - 1))]}"
done
valid=1
fi
else
for i in "${!names[@]}"; do
if [ "${names[$i],,}" = "${token,,}" ]; then
picked+=",${names[$i]}"; valid=1; break
fi
done
fi
[ -n "$valid" ] || warn "ignored unknown selection: $token"
done
if [ -n "$picked" ]; then
ONLY="${picked#,}"
log "selected: ${ONLY//,/ }"
return
fi
done
}
select_packages
# ── the venv ──────────────────────────────────────────────────────────
if [ ! -x "$VENV_DIR/bin/python" ]; then
log "creating venv at $VENV_DIR"
python3 -m venv "$VENV_DIR"
fi
PIP=("$VENV_DIR/bin/pip")
"${PIP[@]}" install -q --upgrade pip setuptools wheel
# ── repository sync: absent | corrupt | dirty | unchanged | moved ────
# Prints the resulting state; never destroys uncommitted work.
sync_repo() {
local name="$1" url="$2"
local dir="$GIT_MIRROR/$name"
if [ ! -e "$dir" ]; then
local stage="$dir.clone.$$"
if git clone --quiet --depth 1 "$url" "$stage"; then
mv "$stage" "$dir"
echo "cloned"
else
rm -rf "$stage"
echo "clone-failed"
fi
return
fi
if [ ! -d "$dir/.git" ]; then
local aside="$dir.corrupt-$(date +%s)"
warn "[$name] not a git checkout — moved aside to $(basename "$aside")"
mv "$dir" "$aside"
local stage="$dir.clone.$$"
if git clone --quiet --depth 1 "$url" "$stage"; then
mv "$stage" "$dir"
echo "recloned"
else
rm -rf "$stage"
echo "clone-failed"
fi
return
fi
local origin_url
origin_url="$(git -C "$dir" remote get-url origin 2>/dev/null || true)"
if [ "$origin_url" != "$url" ]; then
warn "[$name] remote is '$origin_url', table says '$url' — skipping"
echo "url-mismatch"
return
fi
if [ -n "$(git -C "$dir" status --porcelain 2>/dev/null)" ]; then
warn "[$name] local modifications — not touching them:"
git -C "$dir" status --porcelain | sed 's/^/ /'
echo "dirty"
return
fi
local branch refspec
branch="$(git -C "$dir" rev-parse --abbrev-ref HEAD)"
if [ "$branch" = "HEAD" ]; then
refspec="HEAD" # detached shallow clone
else
refspec="$branch"
fi
if ! git -C "$dir" fetch --quiet --depth 1 origin "$refspec"; then
warn "[$name] fetch failed (offline?)"
echo "unchanged"
return
fi
local head fetched
head="$(git -C "$dir" rev-parse HEAD)"
fetched="$(git -C "$dir" rev-parse FETCH_HEAD)"
if [ "$head" = "$fetched" ]; then
echo "unchanged"
return
fi
git -C "$dir" reset --hard --quiet "$fetched"
echo "moved"
}
module_importable() {
"$VENV_DIR/bin/python" -c "import $1" >/dev/null 2>&1
}
# ── per-package install with step-down ────────────────────────────────
install_one() {
local name="$1" url="$2" pipname="$3" extras="$4" method="$5" module="$6"
local dir="$GIT_MIRROR/$name"
if [ -n "$ONLY" ]; then
case ",$ONLY," in *",$name,"*) ;; *) return 0 ;; esac
fi
local state
state="$(sync_repo "$name" "$url")"
case "$state" in
clone-failed)
warn "[$name] clone failed — skipping"
return 0 ;;
url-mismatch | dirty)
return 0 ;; # never clobber; report above
esac
if [ "$UPDATE_ONLY" -eq 1 ]; then
log "[$name] $state"
return 0
fi
# Unchanged checkout + already importable: nothing to do.
if [ "$state" = "unchanged" ] && module_importable "$module"; then
log "[$name] up to date ($(git -C "$dir" rev-parse --short HEAD))"
return 0
fi
if [ "$method" = "source" ] || [ "$FORCE_SOURCE" -eq 1 ]; then
log "[$name] $state — installing from source checkout"
# Local-path extras: pip accepts "<dir>[extra1,extra2]".
if "${PIP[@]}" install -q "${dir}${extras}"; then
return 0
fi
warn "[$name] source build failed — stepping down to wheel"
fi
log "[$name] $state — installing ${pipname}${extras} (wheel)"
"${PIP[@]}" install -q "${pipname}${extras}"
}
for row in "${PACKAGES[@]}"; do
IFS='|' read -r name url pipname extras method module _ <<<"$row"
install_one "$name" "$url" "$pipname" "$extras" "$method" "$module"
done
# ── thicket itself, editable from this checkout ───────────────────────
log "[thicket] installing project (editable)"
"${PIP[@]}" install -q -e "$PROJECT_ROOT"
# ── launcher in the AI tree ───────────────────────────────────────────
cat > "$BIN_DIR/thicket" <<WRAPPER
#!/usr/bin/env bash
# Thicket launcher — /mnt/AI venv, project checkout, one command.
exec "$VENV_DIR/bin/python" "$PROJECT_ROOT/thicket.py" "\$@"
WRAPPER
chmod +x "$BIN_DIR/thicket"
# PATH command (~/.local/bin) — same venv, same interpreter.
# Nothing is ever written inside the project checkout.
mkdir -p "$HOME/.local/bin"
cat > "$HOME/.local/bin/thicket" <<USERWRAPPER
#!/usr/bin/env bash
# thicket — console/CLI with the project venv's interpreter.
exec "$VENV_DIR/bin/python" "$PROJECT_ROOT/thicket.py" "\$@"
USERWRAPPER
chmod +x "$HOME/.local/bin/thicket"
log "bootstrap complete"
log " venv : $VENV_DIR"
log " launcher : $BIN_DIR/thicket"
log " mirror : $GIT_MIRROR ($(ls "$GIT_MIRROR" | wc -l) checkouts)"
log "verify : $BIN_DIR/thicket --dry-run"
Reference in New Issue View Git Blame Copy Permalink