From d244d69ca62655c6aee26eb1c9cc773bf4339ae3 Mon Sep 17 00:00:00 2001 From: Jeremy Anderson Date: Sun, 27 Sep 2026 02:07:26 -0400 Subject: [PATCH] AI-LSC: AI - Local Stack Control --- .env | 1 + CHANGES.md | 201 +++- QA-PASS-REPORT.md | 63 + README.md | 4 +- ai_lsc.py | 2 +- bootstrap.sh | 12 +- gitcommit | 4 +- pyproject.toml | 2 +- quickstart.md | 4 +- src/ai_lsc/__init__.py | 2 +- src/ai_lsc/agents/librechat_config.py | 2 +- src/ai_lsc/constants.py | 19 +- src/ai_lsc/registry/defaults.py | 32 +- src/ai_lsc/registry/layers/inference.py | 19 +- src/ai_lsc/registry/layers/user_interfaces.py | 112 +- src/ai_lsc/registry/validator.py | 6 +- src/ai_lsc/runtime/executor.py | 74 +- src/ai_lsc/runtime/installer.py | 496 ++++++-- src/ai_lsc/stack/export.py | 1020 ++++++++++++++++- src/ai_lsc/ui/main_window.py | 57 +- src/ai_lsc/ui/pages/container_stacks_tab.py | 50 +- src/ai_lsc/ui/pages/git_worktree_tab.py | 6 +- src/ai_lsc/ui/pages/service_row.py | 9 +- src/ai_lsc/ui/pages/verification_tab.py | 1 + src/ai_lsc/utils/paths.py | 176 ++- 25 files changed, 2217 insertions(+), 157 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..be126dd --- /dev/null +++ b/.env @@ -0,0 +1 @@ +AI_LSC_BASE_DIR=/mnt/AI diff --git a/CHANGES.md b/CHANGES.md index e2343de..b3b6e3d 100755 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,204 @@ # AI-LSC Changelog +## v3.4 — File-system map alignment + QCOW2 exports + real updates + +### 1. The base dir now follows the canonical file-system map + +`constants.py` / `utils/paths.py` are the single source of truth for +the /mnt/AI layout, and the runtime code now actually honours it: + +- **Install-root router** (`utils/paths.py::install_dir_for`): a + tool's declared `filesystem.install` is honoured verbatim; when + absent, web UIs (`flags.has_web`, launcher not pinned to + `{tools_root}`) route to `dashboards//`, everything else + stays under `tools//`. `runtime/installer.py` resolves + every per-tool destination through the router. +- **Multi-root detection**: `candidate_install_dirs()` probed by + installer preflight, verification, version detection, and the + main-window drift audit — so tools installed before the routing + change are still detected, verified, and **updated in place** + instead of being orphaned or duplicated. +- **Registry state moved** from `/registry/` (off-map) to + `/configs/registry/` (app state belongs under configs/ per + the map). `main_window._migrate_registry_dir()` migrates existing + installs once on startup; legacy dir removed when emptied. +- **`exports/qcow2/`** added to REQUIRED_DIRS (25 dirs now) alongside + `exports/oci-images/`. +- Ollama server probe order is now map-correct: + `runtime/ollama` → `tools/ollama` → legacy `ollama` / `bin/ollama`. +- Every launcher-template placeholder from the path tree resolves + everywhere (`{dashboards_root}`, `{runtime_root}`, `{configs_root}`, + `{datasets_root}`, …) — in `resolve_launcher_cmd`, the runtime + executor's `format_context`, env overrides, post_install commands, + and the stack-export placeholder chain. + +### 2. QCOW2 / QCOW VM-disk exports (the missing export backend) + +`stack/export.py::ContainerBackend.write_vm_image()` — a fully rootless +pipeline (no loop mounts, no root): + +1. Stage a rootfs tree (`/opt/ai-lsc/stack.json` + compose file, + `/root/start.sh` boot script, `/etc/ai-lsc-release`) under + `exports/qcow2/.staging-*`. +2. Populate an ext4 filesystem straight from the staging dir with + `mkfs.ext4 -d` into a sparse raw image (auto-sized: staging usage + × 4 + headroom, min 2 GiB). +3. `qemu-img convert -c -O qcow2` (also `raw`; legacy `qcow` on + older qemu hosts — modern qemu ≥ ~10 refuses v1 writes and the + exporter surfaces a clean, actionable error suggesting qcow2). +4. Sidecar `.manifest.json` with sha256, virtual size, + `qemu-img info`, tool list, and a boot hint (rootfs disk — pair + with an external kernel such as the Firecracker export's vmlinux, + or attach to any VM). + +Tool detection is built in: missing `qemu-img` / `mkfs.ext4` raise +`FileNotFoundError` with the pacman/apt install hint. UI: two new +buttons on the Container Stacks tab ("Export -> QCOW2 Image", +"Export -> QCOW (legacy)"); the file list shows images with sizes. + +Formats are registry-driven (`VM_IMAGE_FORMATS`, one output dir per +format under `exports/`). + +### 2b. QCrows (.qcrows) VM container image export — the real format + +QCrows (cue-crows) is the in-house **self-describing VM container +image format for Kata Containers**, master-implemented in the +**cockpit-kata** project (`qcrows-spec.md` v0.2.0 + the `qcrows-pack` +/ `qcrows-verify` / `qcrows-inspect` / `qcrows-export` / +`qcrows-initrd-regen` tools). (An earlier iteration of this changelog +misidentified qcrows as sorcery-go's Sovereign Bundle — that format is +now supported under its own `svb` key, below.) + +`ContainerBackend.write_qcrows()` produces a spec-conformant archive: + +``` + (PAX per spec §2.1; .qcrows or .qcrows.gz) + ├── metadata.toml generated manifest (qcrows-pack's template + │ incl. section order qcrows-inspect parses) + ├── menu.toml Cockpit UI menu entry + ├── hashes.sha256 sha256sum -c byte-compatible, ./-prefixed + ├── rootfs.tar.gz the staged ai-lsc stack rootfs + ├── boot-params.conf console=ttyS0 root=/dev/vda rw + ├── build.toml ai-lsc provenance (generator, kernel origin) + ├── spec.md per-image build guide + └── kernel/ REQUIRED (v0.2+): vmlinuz|vmlinux + config +``` + +- **Kernel sourcing** (kernel is REQUIRED by v0.2 — same refusal + behaviour as qcrows-pack): `/runtime/qcrows/` (canonical guest + kernel home: `vmlinuz|vmlinux` + `config`) → the firecracker dir's + `vmlinux` → host kernel fallback (`/boot/vmlinuz-`, any + `/boot/vmlinuz-*` for Arch-style names; config from + `/boot/config-*`, `/usr/lib/modules//config`, or + `/proc/config.gz`). Host fallback logs a warning — a Kata-tuned + guest kernel is recommended for real deployments. +- **Pre-flight Kata checks**: kernel-version probe mirrors + qcrows-pack (bzImage strings → `CONFIG_VERSION_SIGNATURE` → + "unknown"); the five required Kata config options + (`CONFIG_VSOCKETS`, `CONFIG_VIRTIO`, `CONFIG_VIRTIO_PCI`, + `CONFIG_DEVTMPFS`, `CONFIG_DEVTMPFS_MOUNT`) are checked and + shortfalls logged + recorded in the sidecar (qcrows-verify warns, + never fails, on these). +- **Master round-trip verified**: images built by ai-lsc pass + cockpit-kata's own `qcrows-verify` (11 PASS / 0 FAIL / exit 0, both + compressed and uncompressed) and render fully in `qcrows-inspect`. + Tar members carry the `./` prefix exactly like qcrows-pack's + `tar czf - ./*` output; hashes match `sha256sum -c` byte-for-byte. +- Output: `exports/qcrows/ai-lsc-stack-.qcrows` (+ optional + `.gz`) with an ai-lsc sidecar manifest (sha256, kernel origin, + config warnings, verify hint). UI button + "Export -> QCrows Image (.qcrows)". + +**Tool quirks found in cockpit-kata** (their code, reported not +modified — the ai-lsc writer avoids all of them): + +1. `qcrows-pack` renames `initrd.cpio.gz` to `initrd.gz` — its + extension grep (`\.[^.]*$`) only captures the last suffix, and the + packer then derives both the on-disk name and `metadata.toml`'s + `initrd.path` from it; `qcrows-verify`'s initrd probe + (`initrd.img|initrd.cpio.gz|…`) then can't find it. +2. `qcrows-pack` writes GNU-format tar although spec §2.1 mandates + PAX (ai-lsc writes PAX). +3. `metadata.toml`'s generated `rootfs.path` assumes the source + tarball is named `rootfs.*` — other names produce a path that + doesn't match the staged file. +4. `qcrows-inspect`'s first-match key parsing reports kernel's + `included`/`size_mb` for the initrd/rootfs rows (visible in its + own output on its own packs). + +### 2c. Sovereign Bundle (.svb) export (sorcery-go format) + +Kept as its own format key (`svb`) after the qcrows identification +was corrected: `ContainerBackend.write_svb()` implements +`sorcery-sovereign-bundle-v1` (METADATA.json first entry, optional +64-byte ed25519 SIGNATURE.sig over the payload sha256, per-file +MANIFEST.txt, then payload) with deterministic `payload_sha` +(identical content → identical hash) and optional signing via +`AI_LSC_SVB_SIGNING_KEY` + the `cryptography` package. Output: +`exports/svb/*.svb`. Cross-verified with a Go verifier mirroring +sorcery-go's parse path — including the discovery that sorcery-go's +own `VerifySVBSignature` diverges from its builder (it hashes +concatenated entry contents instead of the inner tar.gz), so their +verifier can't validate their own bundles; the ai-lsc writer follows +the builder. + +### 3. Updates actually update now + +The registry's `installer.update_cmd` existed in the schema since +v3.0 but nothing executed it. New +`InstallerManager.update_tool()`: + +- Registry `update_cmd` runs first, rendered with all path-tree + placeholders plus `{install_dir}`, cwd = the *detected* install + location (so pre-routing installs update where they live). +- Per-type defaults otherwise: `uv tool upgrade` (reinstall + step-down preserves `--with` extras), `pipx upgrade`, per-tool + venv `pip install -U`, `git pull --ff-only` (+ yarn rebuild for + git_node), `npm update`. +- `install_with_preflight()` now flows found-tools into + `update_tool()` by default — the UI's single Deploy/Update button + is a real in-place updater instead of returning "already + installed; use force=True". +- New `RuntimeExecutor.update_tool()` facade; service rows pass the + full registry entry through (`registry_entry=`) so routing, + update_cmd, and post_install all flow from one source. + +### 4. Open WebUI install/start fixes + +- **Step-down data loss fixed**: `install_uv` used to fall back to + `pipx install open-webui` — silently dropping `--with + audioop-lts`, producing an install that later crashed importing + `audioop` on Python 3.13. The chain is now: uv (default Python) + → uv with a managed CPython 3.12 (`uv` fetches it automatically — + sidesteps wheels not yet rebuilt for newer Pythons) → per-tool pip + venv installing the package **plus every `--with` extra**. +- Launcher pre-creates `workspaces/openwebui` (DATA_DIR) alongside + the existing `configs/auth` bootstrap so first start can't fail on + a missing data dir. + +### 5. LibreChat install/start fixes + +- **Fresh-clone start bug**: `yarn backend` aborts without `.env`, + and production mode needs `yarn build` assets that plain + `yarn install` never produces. The installer entry now carries + `post_install: "cp -n .env.example .env 2>/dev/null || true; + yarn build"` — runs after clone/update in the project dir. +- **Map-aligned paths**: installs to `dashboards/librechat/` + (was `tools/librechat`), data → `workspaces/librechat` (the + off-map root-level `data/` dir is gone), config stays + `configs/librechat`, launcher uses `{dashboards_root}`. Existing + `tools/librechat` clones are detected and updated in place. +- `agents/librechat_config.py` docstring path corrected to the + dashboards location. + +### 6. Housekeeping + +- Version 3.3.0 → 3.4.0 (`pyproject.toml`, `constants.APP_VERSION`). +- Container Stacks tab lists qcow2 images (+ manifests, sizes) and + excludes the qcow2 dir from the generic directory listing. +- Verification tab passes the full registry entry so multi-root + detection applies to compliance checks too. + ## v3.3 — Registry git-URL sync + bandwidth-aware git-pull + ComfyUI Patch release over v3.2. No schema changes, no UI reorganisation, no @@ -123,7 +322,7 @@ bandwidth-saving behaviour the user asked for. - `pyproject.toml`: 3.2.0 → 3.3.0 - `src/ai_lsc/constants.py`: `APP_VERSION` 3.2.0 → 3.3.0 - (`APP_CODENAME` unchanged: `"Decalogue"`) + (`APP_CODENAME` unchanged: `"Ankh of Jah"`) - New file: `docs/REGISTRY-URLS-v3.3.0.md` (per-file old → new URL table, also serves as a release audit trail). - `gitcommit` rewritten for v3.3.0. diff --git a/QA-PASS-REPORT.md b/QA-PASS-REPORT.md index a56b8cf..478321e 100644 --- a/QA-PASS-REPORT.md +++ b/QA-PASS-REPORT.md @@ -187,3 +187,66 @@ WEBUI_DATA_DIR={workspaces_root}/hermes-webui OLLAMA_BASE_URL=http://localhost:1 The ai-lsc runtime resolves env-prefix shells via `shlex.split()` on the launcher cmd, so the env-assignment form is parsed correctly. + +--- + +## 11. Open WebUI Python-3.13 audioop fix (follow-up #2) + +After the launcher fix landed, open-webui started successfully but crashed +at import time inside the uv-managed tool env: + +``` +ModuleNotFoundError: No module named 'audioop' +… +ModuleNotFoundError: No module named 'pyaudioop' +``` + +Root cause: Python 3.13 removed the `audioop` stdlib module (PEP 594). +`pydub` (an open-webui dep) imports `audioop` and falls back to +`pyaudioop` on ImportError; neither was present in the uv-managed +open-webui env at `~/.local/share/uv/tools/open-webui/`. + +### Fix + +`uv tool install` supports `--with ` for additional dependencies. +The `pkg` field in the registry now carries that flag, and `install_uv` +was hardened to shlex-split `pkg` so the flag passes through verbatim: + +- `src/ai_lsc/runtime/installer.py` `install_uv` — shlex-splits `pkg`; + validates only the first token as the package name; passes remaining + tokens as additional argv to `uv tool install`. Falls back to pipx + using just the first token. +- `src/ai_lsc/runtime/installer.py` `_detect_installation` / + `_binary_name` / `_detect_version` — all now take only the first + shlex token of `pkg` when looking up the binary on PATH or building + version-detection commands. (Without this, preflight would look for + a binary literally named `open-webui --with audioop-lts`.) +- `src/ai_lsc/registry/layers/user_interfaces.py` — both `openwebui` + and `hermes_webui` installer.pkg changed from `"open-webui"` to + `"open-webui --with audioop-lts"`. Inline comment explains the + Python-3.13 / PEP-594 rationale. +- `src/ai_lsc/registry/defaults.py` — openwebui entry updated to match. + +### Verification + +- 95 Python files AST-parse cleanly. +- Registry loads 187 tools; `validate_registry()` reports 0 errors. +- `shlex.split("open-webui --with audioop-lts")` yields the expected + argv: `["open-webui", "--with", "audioop-lts"]`. +- `_binary_name("open-webui --with audioop-lts", "uv")` resolves to + `"open_webui"` (the actual on-PATH binary name). + +### Operator workaround for existing installs + +If open-webui was installed before this fix landed, the audioop-lts +backport can be added in place: + +```bash +~/.local/share/uv/tools/open-webui/bin/python -m pip install audioop-lts +``` + +Or reinstall via the new pkg spec: + +```bash +uv tool install --force "open-webui --with audioop-lts" +``` diff --git a/README.md b/README.md index ce7453a..a8e88de 100755 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@

AI - Local Stack Control

- v3.1.1 — Codename: Ankh of Jah (local-coder-mesh build)
+ v3.4.0 — Codename: Ankh of Jah
Author: Jeremy Anderson <info@dcos.net>
Web: https://dcos.net

-

This build includes the local-coder-mesh integration — see CHANGES.md for the full list of changes vs upstream v3.1.

+

Includes the local-coder-mesh integration, 10-layer taxonomy migration, MoE QA pass, and openhuman/openwebui hardening — see CHANGES.md for the full version history.

A PySide6 desktop application for orchestrating local AI/ML tool stacks across a 10-layer architecture. diff --git a/ai_lsc.py b/ai_lsc.py index 4677b03..fc20cd2 100755 --- a/ai_lsc.py +++ b/ai_lsc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""AI Local Stack Control v3.1 — Ankh of Jah +"""AI Local Stack Control v3.3.0 — Ankh of Jah Direct launcher. Run from the project root: diff --git a/bootstrap.sh b/bootstrap.sh index 95f3ae4..02105d4 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # ────────────────────────────────────────────────────────────── -# AI Local Stack Control v3.1.1 — Ankh of Jah +# AI Local Stack Control v3.3.0 — Ankh of Jah # Bootstrap Script # # Fully portable: works wherever the tarball lands. @@ -46,7 +46,7 @@ export AI_LSC_BASE_DIR="$AI_BASE" echo "" echo -e "${BOLD}╔══════════════════════════════════════════════════════╗${NC}" -echo -e "${BOLD}║ AI Local Stack Control v3.1.1 — Ankh of Jah ║${NC}" +echo -e "${BOLD}║ AI Local Stack Control v3.3.0 — Ankh of Jah ║${NC}" echo -e "${BOLD}╚══════════════════════════════════════════════════════╝${NC}" echo "" echo -e "${CYAN} Project root : ${SCRIPT_DIR}${NC}" @@ -316,11 +316,15 @@ else ERRORS=$((ERRORS + 1)) fi -for cmd in ollama podman docker tmux ripgrep fd tree-sitter; do +for cmd in ollama podman docker tmux ripgrep fd tree-sitter qemu-img; do if command -v "$cmd" &>/dev/null; then info "${cmd} — found" else - warn "${cmd} — not found (optional)" + if [ "$cmd" = "qemu-img" ]; then + warn "qemu-img — not found (needed for QCOW2 VM-image exports: pacman -S qemu-img / apt install qemu-utils)" + else + warn "${cmd} — not found (optional)" + fi fi done diff --git a/gitcommit b/gitcommit index c4cd320..d51b150 100755 --- a/gitcommit +++ b/gitcommit @@ -92,7 +92,7 @@ Intentionally NOT touched: - 3 registry-only git tools with real-looking URLs already in place (goose -> block/goose, nvidia_agent_skills -> NVIDIA/agent-skills, picode -> jasonjmcghee/picode.git). - - APP_CODENAME ("Decalogue") and all historical codename + - APP_CODENAME ("Ankh of Jah") and all historical codename references in docstrings / UI labels / README banner / bootstrap banner — left as-is per user's clarification. @@ -102,7 +102,7 @@ removed from the registry. Versioning: - pyproject.toml: 3.2.0 -> 3.3.0 - src/ai_lsc/constants.py: APP_VERSION 3.2.0 -> 3.3.0 - (APP_CODENAME unchanged: "Decalogue") + (APP_CODENAME unchanged: "Ankh of Jah") - docs/REGISTRY-URLS-v3.3.0.md: per-file old -> new URL table. Verification: diff --git a/pyproject.toml b/pyproject.toml index 48da486..1bb0708 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "ai-lsc" -version = "3.3.0" +version = "3.4.0" description = "AI Local Stack Control — PySide6 desktop app for orchestrating local AI/ML tool stacks" readme = "README.md" license = {text = "AGPL-3.0-or-later"} diff --git a/quickstart.md b/quickstart.md index 494cab4..88ca3dd 100755 --- a/quickstart.md +++ b/quickstart.md @@ -73,7 +73,7 @@ print(f'Registry loaded: {len(DEFAULT_REGISTRY)} tools') print(f'Validation errors: {len(errors)}') " -# Expected output (v3.1): +# Expected output (v3.4.0): # Registry loaded: 125 tools # Validation errors: 0 ``` @@ -253,6 +253,6 @@ for f in os.listdir('ai_lsc/registry/layers'): - Try different **Stack Templates** to find the right combination for your workflow - Set up the **Skills Console** to extend your tool capabilities - Use **Code Analysis** to inspect and understand your project dependencies -- Read [CHANGES.md](CHANGES.md) for the full v3.1 changelog +- Read [CHANGES.md](CHANGES.md) for the full version history (v3.0 → v3.4.0) - Read [whatremains.txt](whatremains.txt) for known deferred items (curl|sh installers, etc.) - Read [docs/ADR-002-pipeline-ticker.md](docs/ADR-002-pipeline-ticker.md) and [docs/ADR-003-workspace-tab.md](docs/ADR-003-workspace-tab.md) for the design rationale behind the two new widgets diff --git a/src/ai_lsc/__init__.py b/src/ai_lsc/__init__.py index 3188874..6ac1acf 100755 --- a/src/ai_lsc/__init__.py +++ b/src/ai_lsc/__init__.py @@ -1,5 +1,5 @@ """ -AI Local Stack Control v3.1 — Release codename: Ankh of Jah. +AI Local Stack Control v3.3.0 — Release codename: Ankh of Jah. Extracted from the monolithic ``ai_lsc_v11.py`` in incremental phases. Currently contains: diff --git a/src/ai_lsc/agents/librechat_config.py b/src/ai_lsc/agents/librechat_config.py index 7bbdc5b..3ad3c77 100755 --- a/src/ai_lsc/agents/librechat_config.py +++ b/src/ai_lsc/agents/librechat_config.py @@ -15,7 +15,7 @@ Usage config.set_ollama_endpoint(ollama_port=11434) config.set_litellm_endpoint(litellm_port=4000) config.set_tool_schemas(tool_schemas) - config.save("/mnt/AI/runtime/librechat/config.yaml") + config.save("/mnt/AI/dashboards/librechat") """ from __future__ import annotations diff --git a/src/ai_lsc/constants.py b/src/ai_lsc/constants.py index 0517d83..05b58e8 100755 --- a/src/ai_lsc/constants.py +++ b/src/ai_lsc/constants.py @@ -1,9 +1,9 @@ """ -AI-LSC v3.1 — Application-wide constants. +AI-LSC v3.3.0 — Application-wide constants. Release codename: Ankh of Jah Pure data: file names, schema version, required directories, default ports, status styles, log colours, service licences, tree-skip patterns, -and the reorganized 10-layer navigation layer order. +and the 10-layer navigation layer order. """ import os @@ -15,8 +15,8 @@ BASE_DIR: str = os.environ.get("AI_LSC_BASE_DIR", "/mnt/AI") CANONICAL_BASE_DIR: str = BASE_DIR # ── Filenames ──────────────────────────────────────────────────────────── -APP_VERSION: str = "3.3.0" -APP_CODENAME: str = "Decalogue" +APP_VERSION: str = "3.4.0" +APP_CODENAME: str = "Ankh of Jah" APP_DISPLAY_NAME: str = f"AI - Local Stack Control v{APP_VERSION} - http://dcos.net" CONFIG_FILE: str = "controller_config.json" APP_ICON_FILE: str = "ai-lsc-logo.png" @@ -51,6 +51,8 @@ REQUIRED_DIRS: list[str] = [ "dashboards", # web UIs and landing pages (Dashy, Open-WebUI, …) "tools", # standalone compiles (built from distfiles) "exports/oci-images", # finalized Podman .tar snapshots (-> MinIO registry) + "exports/qcow2", # VM disk-image exports (qemu qcow2 / legacy qcow) + "exports/qcrows", # QCrows VM container images (.qcrows, spec v0.2) "scripts", # system admin / maintenance automation for the stack "logs", # system, runtime, and pipeline service logs ] @@ -158,11 +160,14 @@ NAV_LAYER_ORDER: list[str] = [ ] # ── Ollama server candidate paths (probed in order) ──────────────── +# Map-aligned order: runtime/ (native binaries) is probed first; +# the legacy root-level and bin/ locations remain as fallbacks for +# pre-v3.4 installs. OLLAMA_SERVER_CANDIDATES: list[str] = [ - "ollama", # /mnt/AI/ollama + "runtime/ollama", # /mnt/AI/runtime/ollama (canonical) "tools/ollama", # /mnt/AI/tools/ollama - "runtime/ollama", # /mnt/AI/runtime/ollama - "bin/ollama" # /mnt/AI/bin/ollama + "ollama", # /mnt/AI/ollama (legacy) + "bin/ollama" # /mnt/AI/bin/ollama (legacy) ] # ── Model tier routing (reserved for v4.0 agentic layer) ────────── diff --git a/src/ai_lsc/registry/defaults.py b/src/ai_lsc/registry/defaults.py index 2639263..82d96ab 100755 --- a/src/ai_lsc/registry/defaults.py +++ b/src/ai_lsc/registry/defaults.py @@ -796,7 +796,10 @@ DEFAULT_REGISTRY: dict = { }, "launcher": { "type": 'tmux', - "cmd": 'OLLAMA_HOST=0.0.0.0:{port} OLLAMA_MODELS={models_root}/ollama ollama serve', + # Force ollama to use /mnt/AI/models/ollama (not ~/.ollama/models) + # and put its identity-key + runtime state under /mnt/AI/.ollama. + # See inference.py ollama entry for full rationale. + "cmd": 'mkdir -p {base_dir}/.ollama && HOME={base_dir} OLLAMA_HOST=0.0.0.0:{port} OLLAMA_MODELS={models_root}/ollama ollama serve', "default_port": 11434, }, "deps": [ @@ -2905,19 +2908,25 @@ DEFAULT_REGISTRY: dict = { "type": 'git_node', "pkg": 'https://github.com/danny-avila/LibreChat.git', "update_cmd": 'git pull --ff-only && yarn install && yarn build', + # Two known fresh-clone start bugs, fixed at install time: + # 1. `yarn backend` aborts without an .env file — seed one + # from the shipped example (never clobber an existing one). + # 2. production mode requires the client assets from + # `yarn build` — plain `yarn install` doesn't produce them. + "post_install": 'cp -n .env.example .env 2>/dev/null || true; yarn build', }, "launcher": { "type": 'tmux', - "cmd": 'cd {tools_root}/librechat && API_PLUGINS=false PORT={port} NODE_ENV=production yarn backend', + "cmd": 'cd {dashboards_root}/librechat && API_PLUGINS=false PORT={port} NODE_ENV=production yarn backend', "default_port": 3080, }, "deps": [ 'ollama', ], "filesystem": { - "install": 'tools/librechat', + "install": 'dashboards/librechat', "config": 'configs/librechat', - "data": 'data/librechat', + "data": 'workspaces/librechat', "logs": 'logs/librechat', }, "description": 'Default agent interface with native tool execution, system files parsing, and multi-user configurations.', @@ -3125,21 +3134,30 @@ DEFAULT_REGISTRY: dict = { "category": 'Extensible Interface', "installer": { "type": 'uv', - "pkg": 'open-webui', + # open-webui's pydub dep imports `audioop`, which Python 3.13 + # removed from the stdlib (PEP 594). The `audioop-lts` backport + # restores it; --with pulls it into the tool's isolated env. + "pkg": 'open-webui --with audioop-lts', "env_overrides": { 'OPEN_WEBUI_CONFIG_DIR': '{base_dir}/configs/openwebui', }, }, "launcher": { "type": 'tmux', - "cmd": 'WEBUI_DATA_DIR={workspaces_root}/openwebui open-webui serve --port {port}', + # Minimal launcher — see user_interfaces.py openwebui entry for + # the full rationale. Summary: open-webui auto-discovers ollama + # at http://localhost:11434 (do NOT set OLLAMA_BASE_URL here). + # DATA_DIR replaces the removed --data-dir CLI flag. cd to + # configs/auth/ so the auto-generated .webui_secret_key lands + # in the canonical private-data dir, not the ai-lsc run dir. + "cmd": 'mkdir -p {base_dir}/configs/auth {workspaces_root}/openwebui && chmod 700 {base_dir}/configs/auth && cd {base_dir}/configs/auth && DATA_DIR={workspaces_root}/openwebui open-webui serve --port {port}', "default_port": 8080, }, "deps": [ 'ollama', ], "filesystem": { - "install": 'tools/openwebui', + "install": 'dashboards/openwebui', "config": 'configs/openwebui', "data": 'workspaces/openwebui', "logs": 'logs/openwebui', diff --git a/src/ai_lsc/registry/layers/inference.py b/src/ai_lsc/registry/layers/inference.py index f86a3bb..01b2da0 100755 --- a/src/ai_lsc/registry/layers/inference.py +++ b/src/ai_lsc/registry/layers/inference.py @@ -35,7 +35,24 @@ TOOLS: dict[str, dict] = { }, "launcher": { "type": "tmux", - "cmd": "OLLAMA_HOST=0.0.0.0:{port} OLLAMA_MODELS={models_root}/ollama ollama serve", + # Force ollama to use /mnt/AI/models/ollama (not ~/.ollama/models) + # and put its identity-key + runtime state under /mnt/AI/.ollama + # (not ~/.ollama). We set HOME={base_dir} *only for ollama* — + # this is safe because ollama doesn't depend on uv-tool/pipx + # (which are $HOME-relative). For uv-tool-installed tools like + # open-webui, we CANNOT override HOME globally or the tool's + # own venv lookup breaks; each tool that needs a HOME redirect + # must opt in via its own launcher cmd. + # + # OLLAMA_HOST=0.0.0.0:{port} — bind to all interfaces so + # open-webui and other tools on the box can reach it. + # OLLAMA_MODELS — pins the models dir to the canonical ai-lsc + # location; this is what makes `ollama list` and open-webui's + # model picker see the same set. + # mkdir -p {base_dir}/.ollama — pre-creates the dotdir ollama + # expects so it doesn't trip on a broken ~/.ollama state on + # the host (e.g. a stale file where a dir should be). + "cmd": "mkdir -p {base_dir}/.ollama && HOME={base_dir} OLLAMA_HOST=0.0.0.0:{port} OLLAMA_MODELS={models_root}/ollama ollama serve", "default_port": 11434 }, "deps": [], diff --git a/src/ai_lsc/registry/layers/user_interfaces.py b/src/ai_lsc/registry/layers/user_interfaces.py index 86f5769..fbe723e 100755 --- a/src/ai_lsc/registry/layers/user_interfaces.py +++ b/src/ai_lsc/registry/layers/user_interfaces.py @@ -20,16 +20,46 @@ TOOLS: dict[str, dict] = { "category": 'Extensible Interface', "installer": { "type": "uv", - "pkg": "open-webui" + # open-webui's pydub dep imports `audioop`, which Python 3.13 + # removed from the stdlib (PEP 594). The `audioop-lts` backport + # restores it; --with pulls it into the tool's isolated env. + "pkg": "open-webui --with audioop-lts" }, "launcher": { + # Minimal launcher — open-webui auto-discovers ollama at + # http://localhost:11434 (its built-in default). Do NOT add + # OLLAMA_BASE_URL / CORS_ORIGINS / ENABLE_DIRECT_CONNECTIONS / + # WEBUI_SECRET_KEY env vars — past iterations proved they break + # model discovery. open-webui handles all of these itself when + # left alone. + # + # Two things we DO control: + # 1. DATA_DIR — points the sqlite db + uploads at the + # canonical workspaces dir. (The old --data-dir CLI flag + # was removed upstream; the env var is the replacement.) + # 2. cd to {base_dir}/configs/auth/ before launching — + # open-webui writes its auto-generated .webui_secret_key + # to Path.cwd() (see backend/open_webui/__init__.py + # KEY_FILE). Running from configs/auth/ keeps the secret + # out of the ai-lsc run dir and in the canonical + # private-data location, with no env-var games. "type": "tmux", - "cmd": "WEBUI_DATA_DIR={workspaces_root}/openwebui open-webui serve --port {port}", + "cmd": "mkdir -p {base_dir}/configs/auth {workspaces_root}/openwebui && chmod 700 {base_dir}/configs/auth && cd {base_dir}/configs/auth && DATA_DIR={workspaces_root}/openwebui open-webui serve --port {port}", "default_port": 8080 }, "deps": [ "ollama" ], + "filesystem": { + # Map-aligned per the v3.4 file-system map: the web UI app + # marker lives under dashboards/, config under configs/, and + # the sqlite/uploads DATA_DIR under workspaces/. The uv-managed + # binary itself lives in tools/.uv/bin (on the managed PATH). + "install": "dashboards/openwebui", + "config": "configs/openwebui", + "data": "workspaces/openwebui", + "logs": "logs/openwebui" + }, "description": "Extensible frontend for LLMs.", "license": 'MIT', "flags": { @@ -79,11 +109,17 @@ TOOLS: dict[str, dict] = { "installer": { "type": "git_node", "pkg": "https://github.com/danny-avila/LibreChat.git", - "update_cmd": "git pull --ff-only && yarn install && yarn build" + "update_cmd": "git pull --ff-only && yarn install && yarn build", + # Two known fresh-clone start bugs, fixed at install time: + # 1. `yarn backend` aborts without an .env file — seed one + # from the shipped example (never clobber an existing one). + # 2. production mode requires the client assets from + # `yarn build` — plain `yarn install` doesn't produce them. + "post_install": "cp -n .env.example .env 2>/dev/null || true; yarn build" }, "launcher": { "type": "tmux", - "cmd": "cd {tools_root}/librechat && API_PLUGINS=false PORT={port} NODE_ENV=production yarn backend", + "cmd": "cd {dashboards_root}/librechat && API_PLUGINS=false PORT={port} NODE_ENV=production yarn backend", "default_port": 3080 }, "deps": [ @@ -101,9 +137,9 @@ TOOLS: dict[str, dict] = { "is_skills_collection": False }, "filesystem": { - "install": "tools/librechat", + "install": "dashboards/librechat", "config": "configs/librechat", - "data": "data/librechat", + "data": "workspaces/librechat", "logs": "logs/librechat" } }, @@ -536,11 +572,21 @@ TOOLS: dict[str, dict] = { "category": 'Chat Frontend', "installer": { "type": "uv", - "pkg": "open-webui" + # Same pydub/audioop Python-3.13 fix as the openwebui entry. + "pkg": "open-webui --with audioop-lts" }, "launcher": { "type": "tmux", - "cmd": "WEBUI_DATA_DIR={workspaces_root}/hermes-webui OLLAMA_BASE_URL=http://localhost:17051 open-webui serve --port {port}", + # Same minimal pattern as the openwebui entry — cd to + # configs/auth/ so .webui_secret_key lands there, DATA_DIR + # points at the hermes-webui data volume. + # + # The ONE env var we set here: OLLAMA_BASE_URL. The default is + # http://localhost:11434 (ollama direct); hermes_webui needs to + # point at hermes_agent (17051) instead so every conversation + # flows through the Hermes runtime's tool-use layer. This is + # the only difference from the openwebui entry. + "cmd": "mkdir -p {base_dir}/configs/auth && chmod 700 {base_dir}/configs/auth && cd {base_dir}/configs/auth && DATA_DIR={workspaces_root}/hermes-webui OLLAMA_BASE_URL=http://127.0.0.1:17051 open-webui serve --port {port}", "default_port": 8081 }, "deps": [ @@ -625,4 +671,54 @@ TOOLS: dict[str, dict] = { "is_skills_collection": False } }, + 'openhuman': { + "name": "OpenHuman", + "level": 10, + "layer": 'Human Interface & System Operations', + "role": 'Personal AI Harness', + "category": 'Personal AI Harness', + "installer": { + # Same curl|sh policy category as ollama / meilisearch / fabric + # (see whatremains.txt C-05). The openhuman README explicitly + # warns this script path is unverified (no detached signature); + # the native-package paths (deb / rpm / AppImage / brew cask) + # are preferred when the user is ready to move off the script + # installer. + "type": "script", + "cmd": "curl -fsSL https://raw.githubusercontent.com/tinyhumansai/openhuman/main/scripts/install.sh | bash", + "pkg": "https://github.com/tinyhumansai/openhuman" + }, + "launcher": { + # Desktop GUI app (Rust core + native shell). One-click start + # → talks to localhost ollama. OLLAMA_BASE_URL is preset so + # openhuman's first-run model-router defaults to the local + # ollama instance instead of the managed subscription endpoint. + # Wrapped in `bash -c` because launch_desktop uses + # subprocess.Popen(argv) directly (no shell), so env-assignment + # prefixes must be expanded by an actual shell. + "type": "desktop", + "cmd": "bash -c 'OLLAMA_BASE_URL=http://127.0.0.1:11434 exec openhuman'", + "default_port": None + }, + "deps": [ + "ollama" + ], + "description": "Personal AI harness: persistent local memory tree, " + "agent orchestration on durable graphs, deep-research " + "pipeline. Local-first with opt-in Privacy Mode that " + "forces all inference to localhost ollama (or another " + "managed backend). Native packages (.deb / .rpm / " + ".AppImage / brew cask) preferred over the script " + "installer once the user moves past first boot.", + "license": 'GPL-3.0', + "flags": { + "has_cli": False, + "has_gui": True, + "has_web": False, + "is_ollama": False, + "is_passive": False, + "is_mcp": False, + "is_skills_collection": False + } + }, } diff --git a/src/ai_lsc/registry/validator.py b/src/ai_lsc/registry/validator.py index 985269d..035b43f 100755 --- a/src/ai_lsc/registry/validator.py +++ b/src/ai_lsc/registry/validator.py @@ -175,7 +175,11 @@ def _check_entry(tool_id: str, entry: dict[str, Any]) -> list[str]: # Warn if script cmd doesn't contain {tools_root} if itype == "script" and inst.get("cmd"): cmd_str = inst["cmd"] - if "{tools_root}" not in cmd_str and tool_id != "ollama": + # carve-out: ollama and openhuman ship native installers + # (apt/dpkg/AppImage/brew cask) that target system paths, + # not tools_root. Their script installers are accepted as-is + # per the C-05 curl|sh policy (see whatremains.txt). + if "{tools_root}" not in cmd_str and tool_id not in {"ollama", "openhuman"}: errors.append( f"{tool_id}: script installer cmd should reference " f"{{{{tools_root}}}} to avoid polluting system dirs" diff --git a/src/ai_lsc/runtime/executor.py b/src/ai_lsc/runtime/executor.py index e5fa952..4145dee 100755 --- a/src/ai_lsc/runtime/executor.py +++ b/src/ai_lsc/runtime/executor.py @@ -104,16 +104,28 @@ class RuntimeExecutor: port: str = "", model_arg: str = "", ) -> dict[str, str]: - """Build the ``{placeholders}`` dict used by launcher commands.""" + """Build the ``{placeholders}`` dict used by launcher commands. + + Every key of the path tree is exposed (``{tools_root}``, + ``{dashboards_root}``, ``{runtime_root}``, ``{configs_root}``, + ``{datasets_root}``, …) so registry launcher templates can + reference any canonical /mnt/AI location. + """ from ai_lsc.constants import BASE_DIR - return { + from ai_lsc.utils.paths import build_path_tree + ctx = { + str(k): str(v) + for k, v in build_path_tree(BASE_DIR).items() + } + ctx.update({ "base_dir": BASE_DIR, "tools_root": self.tools_root, "models_root": self.models_root, "workspaces_root": self.workspaces_root, "port": port, "model_arg": model_arg, - } + }) + return ctx # -- service lifecycle ----------------------------------------------- @@ -224,15 +236,26 @@ class RuntimeExecutor: env_overrides: dict[str, str] | None = None, filesystem: dict[str, str] | None = None, license_spdx: str | None = None, + registry_entry: dict | None = None, ) -> str: """Dispatch tool installation to the correct installer. - If *tool_id* is provided, the installer uses preflight detection - and routes artifacts to ``tools_root//``. + If *tool_id* is provided, the installer uses preflight detection. + When the tool is already installed, the call becomes an in-place + **update** (honouring the registry's ``installer.update_cmd``) + unless *force* is set or the tool is missing. + If *force* is True, skips preflight and installs unconditionally. - *post_install* runs a shell command inside ``tools_root/`` - after clone (e.g. ``pip install -r requirements.txt``, ``make``). + *registry_entry* (the tool's full registry metadata dict) routes + artifacts to the canonical install root declared by + ``filesystem.install`` / the v3.4 router (web UIs → + ``dashboards//``, standalone compiles → ``tools//``) + and supplies ``update_cmd`` / ``post_install`` defaults. + + *post_install* runs a shell command inside the tool's install + directory after clone (e.g. ``pip install -r requirements.txt``, + ``make``). *env_overrides* remaps upstream environment variables (HF_HOME, TRANSFORMERS_CACHE, etc.) into ``/mnt/AI/`` paths. @@ -259,6 +282,7 @@ class RuntimeExecutor: post_install=post_install, env_overrides=env_overrides, license_spdx=license_spdx, + entry=registry_entry, ) return self._installer.run( inst_type=inst_type, @@ -269,6 +293,40 @@ class RuntimeExecutor: post_install=post_install, env_overrides=env_overrides, license_spdx=license_spdx, + entry=registry_entry, + ) + + def update_tool( + self, + tool_id: str, + inst_type: str, + pkg: str, + cmd: str = "", + update_cmd: str = "", + registry_entry: dict | None = None, + env_overrides: dict[str, str] | None = None, + post_install: str | None = None, + ) -> str: + """Update an already-installed tool in place. + + Honours the registry's ``installer.update_cmd`` when provided; + otherwise falls back to per-type defaults (``uv tool upgrade``, + ``pipx upgrade``, ``git pull --ff-only`` + node rebuild, …). + Runs in the *detected* install location so tools installed + before the v3.4 layout change update where they live. + + Returns a description of the result. + """ + _validate_tool_id(tool_id) + return self._installer.update_tool( + tool_id=tool_id, + inst_type=inst_type, + pkg=pkg, + cmd=cmd, + update_cmd=update_cmd, + entry=registry_entry, + env_overrides=env_overrides, + post_install=post_install, ) # -- verification --------------------------------------------------- @@ -280,6 +338,7 @@ class RuntimeExecutor: pkg: str, cmd: str = "", filesystem: dict[str, str] | None = None, + registry_entry: dict | None = None, ) -> dict[str, Any]: """Run the installation compliance checklist for a tool. @@ -291,6 +350,7 @@ class RuntimeExecutor: pkg=pkg, cmd=cmd, filesystem=filesystem, + entry=registry_entry, ) # -- model management ------------------------------------------------ diff --git a/src/ai_lsc/runtime/installer.py b/src/ai_lsc/runtime/installer.py index 057cde6..de934be 100755 --- a/src/ai_lsc/runtime/installer.py +++ b/src/ai_lsc/runtime/installer.py @@ -53,6 +53,11 @@ from typing import Any from urllib.parse import urlparse from ai_lsc.utils.logging import get_logger +from ai_lsc.utils.paths import ( + build_path_tree, + candidate_install_dirs, + install_dir_for, +) from ai_lsc.utils.process import enriched_env logger = get_logger(__name__) @@ -92,6 +97,21 @@ def _validate_url(url: str, *, allow_schemes: tuple[str, ...] = ("http", "https" raise ValueError(f"unsafe URL rejected: {url!r}") return url + +def _uv_with_extras(tokens: list[str]) -> list[str]: + """Extract the package names pulled in by ``--with`` flags. + + ``["open-webui", "--with", "audioop-lts"]`` → ``["audioop-lts"]``. + Multiple ``--with`` flags each contribute one package; unknown + flags contribute nothing. Used by the uv→pip step-down so the + fallback venv keeps every extra the registry declared. + """ + extras: list[str] = [] + for i, tok in enumerate(tokens[:-1]): + if tok == "--with": + extras.append(tokens[i + 1]) + return extras + # Step-down containment order (most isolated first) STEP_DOWN_ORDER: list[str] = [ "ollama", "uv", "pipx", "pip", @@ -152,6 +172,43 @@ class InstallerManager: # e.g. in the UI layer). self.license_gate = license_gate + # ── Install-root routing ───────────────────────────────────────── + + def _dest_for( + self, + tool_id: str, + entry: dict[str, Any] | None = None, + ) -> str: + """Resolve the on-disk destination directory for *tool_id*. + + Honours the registry entry's ``filesystem.install`` and the + v3.4 install-root router (web UIs → ``dashboards/``, everything + else → ``tools/``). Falls back to the historical + ``tools_root/`` when no entry is supplied, so + pre-v3.4 call sites keep working unchanged. + """ + if entry: + return str(install_dir_for(entry, tool_id, self.base_dir)) + return os.path.join(self.tools_root, tool_id) + + def _existing_dir_for( + self, + tool_id: str, + entry: dict[str, Any] | None = None, + ) -> str: + """Like :meth:`_dest_for`, but prefer an existing install. + + Scans the candidate roots in router order and returns the first + directory that already exists; only when none exists is the + canonical (fresh-install) destination returned. This keeps + updates flowing to wherever a tool actually lives today + instead of forking a second copy under the new canonical root. + """ + for cand in candidate_install_dirs(tool_id, entry, self.base_dir): + if cand.is_dir(): + return str(cand) + return self._dest_for(tool_id, entry) + # ── Environment construction ───────────────────────────────────── def _env( @@ -198,14 +255,20 @@ class InstallerManager: env["PIPX_HOME"] = os.path.join(self.tools_root, ".pipx") # ── Per-tool env overrides from registry ──────────────────────── - # Keys may contain {tools_root}, {base_dir} placeholders. + # Values may contain any {path_tree_key} placeholder + # ({tools_root}, {base_dir}, {dashboards_root}, …). if env_overrides: + tree = build_path_tree(self.base_dir) for key, raw_val in env_overrides.items(): expanded = raw_val.replace( "{tools_root}", self.tools_root, ).replace( "{base_dir}", self.base_dir, ) + for tree_key, tree_val in tree.items(): + expanded = expanded.replace( + "{" + tree_key + "}", str(tree_val), + ) env[key] = expanded logger.debug( "env override: %s=%s (tool %s)", key, expanded, tool_id, @@ -230,6 +293,7 @@ class InstallerManager: inst_type: str, pkg: str, cmd: str = "", + entry: dict[str, Any] | None = None, ) -> dict[str, Any]: """Check whether a tool is already installed before installing. @@ -246,7 +310,7 @@ class InstallerManager: } location, version = self._detect_installation( - tool_id, inst_type, pkg, cmd, + tool_id, inst_type, pkg, cmd, entry, ) if location: result["found"] = True @@ -262,39 +326,54 @@ class InstallerManager: inst_type: str, pkg: str, cmd: str = "", + entry: dict[str, Any] | None = None, ) -> tuple[str, str]: - """Detect existing installation. Returns (location, version).""" + """Detect existing installation. Returns (location, version). - # 1. Check tools_root/ directory existence - tool_dir = os.path.join(self.tools_root, tool_id) - if os.path.isdir(tool_dir): - ver = self._detect_version(inst_type, pkg, cmd, tool_dir) - return tool_dir, ver + Probes every candidate install root (router dir, dashboards/, + tools/) so tools installed before the v3.4 routing change are + still found and offered an update instead of a reinstall. + """ + + # `pkg` may carry extra uv/flags tokens (e.g. + # "open-webui --with audioop-lts"); for preflight we only care + # about the first token — the actual package name. + pkg_token = shlex.split(pkg)[0] if pkg else pkg + + # 1. Check candidate install directories (router order) + for tool_dir in candidate_install_dirs( + tool_id, entry, self.base_dir, + ): + if os.path.isdir(tool_dir): + ver = self._detect_version( + inst_type, pkg_token, cmd, str(tool_dir), + ) + return str(tool_dir), ver # 2. Check tools_root/.pipx, tools_root/.uv, tools_root/.local for subdir in [".pipx", ".uv", ".local"]: - check = os.path.join(self.tools_root, subdir, "bin", pkg) + check = os.path.join(self.tools_root, subdir, "bin", pkg_token) if os.path.exists(check): return os.path.dirname(check), "" # 3. Check tools_root/bin - bin_check = os.path.join(self.tools_root, "bin", pkg) + bin_check = os.path.join(self.tools_root, "bin", pkg_token) if os.path.exists(bin_check): return os.path.dirname(bin_check), "" # 4. Check system PATH via shutil.which - binary_name = self._binary_name(pkg, inst_type) + binary_name = self._binary_name(pkg_token, inst_type) system_path = shutil.which(binary_name) if system_path: - ver = self._detect_version(inst_type, pkg, cmd) + ver = self._detect_version(inst_type, pkg_token, cmd) return system_path, ver # 5. OS package manager query (pacman / dnf / apt) — list-form # subprocess calls, no shell, no interpolation. _PKG_MGR_QUERIES: dict[str, list[str]] = { - "pacman": ["pacman", "-Qi", pkg], - "dnf": ["dnf", "info", pkg], - "apt": ["dpkg", "-s", pkg], + "pacman": ["pacman", "-Qi", pkg_token], + "dnf": ["dnf", "info", pkg_token], + "apt": ["dpkg", "-s", pkg_token], } if inst_type in _PKG_MGR_QUERIES: try: @@ -306,19 +385,24 @@ class InstallerManager: for line in proc.stdout.splitlines(): if line.strip().startswith("Version"): ver = line.split(":", 1)[-1].strip() - return f"{inst_type}:{pkg}", ver + return f"{inst_type}:{pkg_token}", ver except (OSError, subprocess.SubprocessError): pass return "", "" def _binary_name(self, pkg: str, inst_type: str) -> str: - """Map a package name to its likely binary name.""" + """Map a package name to its likely binary name. + + Takes the first shlex token of *pkg* so a spec like + ``"open-webui --with audioop-lts"`` resolves to ``open_webui``. + """ + first = shlex.split(pkg)[0] if pkg else pkg if inst_type == "npm": - return pkg if "/" not in pkg else pkg.split("/")[-1] + return first if "/" not in first else first.split("/")[-1] if inst_type in ("uv", "pip"): - return pkg.replace("-", "_").replace(".", "_") - return pkg + return first.replace("-", "_").replace(".", "_") + return first def _detect_version( self, @@ -329,9 +413,19 @@ class InstallerManager: ) -> str: """Try to extract the installed version.""" if inst_type == "git": - git_dir = os.path.join(self.tools_root, pkg.split("/")[-1] - .replace(".git", "")) - if os.path.isdir(os.path.join(git_dir, ".git")): + # Use only the first token of `pkg` for the dir-name lookup + # so a spec like "repo --branch foo" doesn't break the + # rsplit("/"). (Git pkg specs are normally bare URLs, but + # be defensive.) + pkg_first = shlex.split(pkg)[0] if pkg else pkg + repo_name = pkg_first.split("/")[-1].replace(".git", "") + git_dirs = [ + str(d) for d in candidate_install_dirs( + repo_name, None, self.base_dir, + ) + if os.path.isdir(os.path.join(d, ".git")) + ] + for git_dir in git_dirs: for argv in ( ["git", "describe", "--tags", "--abbrev=0"], ["git", "rev-parse", "--short", "HEAD"], @@ -355,7 +449,11 @@ class InstallerManager: else: tmpl = _VERSION_CMDS.get(inst_type, "") if tmpl: - ver_argv = shlex.split(tmpl.format(pkg=pkg, cmd=pkg)) + # Use only the first token of `pkg` so a spec like + # "open-webui --with audioop-lts" renders as just the + # binary name in the version-detection command. + pkg_first = shlex.split(pkg)[0] if pkg else pkg + ver_argv = shlex.split(tmpl.format(pkg=pkg_first, cmd=pkg_first)) if not ver_argv: return "" @@ -378,14 +476,16 @@ class InstallerManager: self, tool_id: str, post_install_cmd: str, + dest: str = "", ) -> str: - """Run a post-install hook inside ``tools_root/``.""" + """Run a post-install hook inside the tool's install directory.""" if not post_install_cmd: return "" - dest = os.path.join(self.tools_root, tool_id) + cwd = dest or os.path.join(self.tools_root, tool_id) env = self._env(tool_id) - # Replace {tools_root} in the command - cmd = post_install_cmd.replace("{tools_root}", self.tools_root) + # Resolve {tools_root} / {base_dir} / {install_dir} and every + # path-tree placeholder in the command. + cmd = self._render_cmd(post_install_cmd, cwd) logger.info("Running post-install for %s: %s", tool_id, cmd) try: # Post-install commands are arbitrary shell snippets supplied by @@ -395,7 +495,7 @@ class InstallerManager: # break out of the subprocess call itself. subprocess.run( ["bash", "-c", cmd], check=True, env=env, - timeout=300, cwd=dest, + timeout=600, cwd=cwd, ) return f"Post-install completed for {tool_id}." except (subprocess.CalledProcessError, OSError) as exc: @@ -404,6 +504,20 @@ class InstallerManager: ) return f"Post-install FAILED for {tool_id}: {exc}" + def _render_cmd(self, cmd: str, install_dir: str = "") -> str: + """Resolve path placeholders in a registry shell snippet. + + Supports ``{tools_root}``, ``{base_dir}``, ``{install_dir}`` + and every key of the path tree (``{dashboards_root}``, …). + """ + rendered = cmd.replace("{tools_root}", self.tools_root) + rendered = rendered.replace("{base_dir}", self.base_dir) + if install_dir: + rendered = rendered.replace("{install_dir}", install_dir) + for key, value in build_path_tree(self.base_dir).items(): + rendered = rendered.replace("{" + key + "}", str(value)) + return rendered + # ── Strategy methods ──────────────────────────────────────────── def install_ollama(self, pkg: str, tool_id: str) -> str: @@ -438,26 +552,84 @@ class InstallerManager: return f"Ollama model '{pkg}' queued for pull." def install_uv(self, pkg: str, tool_id: str, - env_overrides: dict[str, str] | None = None) -> str: - """Install a Python tool via ``uv tool install`` pinned to tools_root.""" - dest = os.path.join(self.tools_root, tool_id) + env_overrides: dict[str, str] | None = None, + entry: dict[str, Any] | None = None) -> str: + """Install a Python tool via ``uv tool install`` pinned to tools_root. + + The ``pkg`` string may carry extra ``uv`` flags after the package + name — e.g. ``"open-webui --with audioop-lts"``. The first token + is validated as the package name; the remaining tokens are passed + through to ``uv tool install`` verbatim. This lets registry + entries pin Python-3.13-deps backports (audioop-lts for pydub, + etc.) without needing new schema fields or threading new params + through every caller. + + Step-down containment (fix for the open-webui install bug): + + 1. ``uv tool install [flags]`` with the default Python. + 2. If that fails, retry with ``--python 3.12`` — uv fetches a + managed CPython 3.12 on the fly, which sidesteps wheels that + have not been rebuilt for the host's newer Python (the + pydub/audioop class of failure). + 3. If uv fails entirely, fall back to a per-tool pip venv that + installs the package **plus every ``--with`` extra** — the + historical pipx fallback silently dropped the extras, which + produced installs that broke at import time. + """ + dest = self._dest_for(tool_id, entry) os.makedirs(dest, exist_ok=True) env = self._env(tool_id, env_overrides) + # Shlex-split so registry authors can write + # `pkg: "open-webui --with audioop-lts"` and have it land as + # `["uv", "tool", "install", "open-webui", "--with", "audioop-lts"]`. + tokens = shlex.split(pkg) + if not tokens: + raise ValueError(f"invalid pkg spec for {tool_id}: {pkg!r}") + _validate_pkg(tokens[0]) + extras = _uv_with_extras(tokens) try: - _validate_pkg(pkg) subprocess.run( - ["uv", "tool", "install", pkg], + ["uv", "tool", "install", *tokens], check=True, env=env, timeout=300, ) - return f"UV tool '{pkg}' installed to {env['UV_TOOL_DIR']}." + return f"UV tool '{tokens[0]}' installed to {env['UV_TOOL_DIR']}." except subprocess.CalledProcessError: - logger.info("uv install failed for %s, stepping down to pipx", pkg) - return self.install_pipx(pkg, tool_id, env_overrides) + logger.info( + "uv install failed for %s on the default Python; " + "retrying with a managed CPython 3.12", tokens[0], + ) + try: + subprocess.run( + ["uv", "tool", "install", "--python", "3.12", *tokens], + check=True, env=env, timeout=600, + ) + return ( + f"UV tool '{tokens[0]}' installed to " + f"{env['UV_TOOL_DIR']} (managed CPython 3.12)." + ) + except subprocess.CalledProcessError: + if extras: + logger.info( + "uv failed for %s even on 3.12; falling back to a " + "pip venv WITH the %d --with extra(s) intact", + tokens[0], len(extras), + ) + pip_spec = " ".join([tokens[0], *extras]) + return self.install_pip( + pip_spec, tool_id, env_overrides, entry, + ) + logger.info( + "uv failed for %s; stepping down to pipx", tokens[0], + ) + return self.install_pipx( + tokens[0], tool_id, env_overrides, entry, + ) def install_pipx(self, pkg: str, tool_id: str, - env_overrides: dict[str, str] | None = None) -> str: + env_overrides: dict[str, str] | None = None, + entry: dict[str, Any] | None = None) -> str: """Install a Python CLI tool via ``pipx`` pinned to tools_root.""" - dest = os.path.join(self.tools_root, tool_id) + dest = self._dest_for(tool_id, entry) os.makedirs(dest, exist_ok=True) env = self._env(tool_id, env_overrides) try: @@ -469,12 +641,18 @@ class InstallerManager: return f"pipx '{pkg}' installed to {env['PIPX_HOME']}." except subprocess.CalledProcessError: logger.info("pipx install failed for %s, stepping down to pip", pkg) - return self.install_pip(pkg, tool_id, env_overrides) + return self.install_pip(pkg, tool_id, env_overrides, entry) def install_pip(self, pkg: str, tool_id: str, - env_overrides: dict[str, str] | None = None) -> str: - """Install a Python tool via ``pip`` into a per-tool venv.""" - dest = os.path.join(self.tools_root, tool_id) + env_overrides: dict[str, str] | None = None, + entry: dict[str, Any] | None = None) -> str: + """Install a Python tool via ``pip`` into a per-tool venv. + + *pkg* may be a space-separated spec naming several distributions + (``"open-webui audioop-lts"``) — the uv step-down path uses this + to keep ``--with`` extras alive in the fallback venv. + """ + dest = self._dest_for(tool_id, entry) venv_dir = os.path.join(dest, ".venv") os.makedirs(dest, exist_ok=True) env = self._env(tool_id, env_overrides) @@ -484,16 +662,20 @@ class InstallerManager: check=True, env=env, timeout=60, ) pip_bin = os.path.join(venv_dir, "bin", "pip") + pip_tokens = shlex.split(pkg) if pkg else [] + if not pip_tokens: + raise ValueError(f"invalid pkg spec for {tool_id}: {pkg!r}") try: - _validate_pkg(pkg) + for tok in pip_tokens: + _validate_pkg(tok) subprocess.run( - [pip_bin, "install", pkg], + [pip_bin, "install", *pip_tokens], check=True, env=env, timeout=300, ) except subprocess.CalledProcessError as exc: logger.warning("pip install failed for %s: %s", pkg, exc) raise - self._symlink_venv_bin(tool_id, venv_dir, pkg) + self._symlink_venv_bin(tool_id, venv_dir, pip_tokens[0]) return f"pip '{pkg}' installed to {venv_dir}." def _symlink_venv_bin( @@ -544,9 +726,10 @@ class InstallerManager: return f"Dispatched apt for {pkg}." def install_npm(self, pkg: str, tool_id: str, - env_overrides: dict[str, str] | None = None) -> str: + env_overrides: dict[str, str] | None = None, + entry: dict[str, Any] | None = None) -> str: """Install an npm package to an isolated prefix under tools_root.""" - dest = os.path.join(self.tools_root, tool_id) + dest = self._dest_for(tool_id, entry) os.makedirs(dest, exist_ok=True) env = self._env(tool_id, env_overrides) _validate_pkg(pkg) @@ -562,6 +745,7 @@ class InstallerManager: tool_id: str, post_install: str | None = None, env_overrides: dict[str, str] | None = None, + entry: dict[str, Any] | None = None, ) -> str: """Clone or update a git repository at ``tools_root/``. @@ -571,8 +755,14 @@ class InstallerManager: fails for any reason — diverged branches, network errors, corrupted index, etc.), move the old dir aside and re-clone fresh, so the install always ends in a usable state. + + The destination follows the v3.4 install-root router when + *entry* is supplied (web UIs → ``dashboards//``). + A pre-existing clone under ``tools//`` is still + detected and updated in place — the install never forks a + second copy just because the routing changed. """ - dest = os.path.join(self.tools_root, tool_id) + dest = self._existing_dir_for(tool_id, entry) git_dir = os.path.join(dest, ".git") # Step-down: each branch resolves one situation and assigns msg. @@ -607,7 +797,7 @@ class InstallerManager: msg = f"Git source cloned: {dest}" if post_install: - self._run_post_install(tool_id, post_install) + self._run_post_install(tool_id, post_install, dest) return msg def install_git_node( @@ -615,6 +805,7 @@ class InstallerManager: pkg: str, tool_id: str, post_install: str | None = None, + entry: dict[str, Any] | None = None, ) -> str: """Clone or update a git repo and run ``yarn install``. @@ -623,8 +814,11 @@ class InstallerManager: then ``yarn install`` to pick up any changed dependencies. If the destination exists but is not a git repo, or if the pull fails, move the old dir aside and re-clone fresh. + + Destination routing matches :meth:`install_git` (existing + clones are updated in place wherever they live). """ - dest = os.path.join(self.tools_root, tool_id) + dest = self._existing_dir_for(tool_id, entry) git_dir = os.path.join(dest, ".git") # Step-down: each branch resolves one situation, runs the @@ -671,7 +865,7 @@ class InstallerManager: msg = f"Git+Node source synchronized: {dest}" if post_install: - self._run_post_install(tool_id, post_install) + self._run_post_install(tool_id, post_install, dest) return msg # ── git install helpers ──────────────────────────────────────── @@ -767,6 +961,7 @@ class InstallerManager: post_install: str | None = None, env_overrides: dict[str, str] | None = None, license_spdx: str | None = None, + entry: dict[str, Any] | None = None, ) -> str: """Dispatch to the correct installer strategy. @@ -774,6 +969,11 @@ class InstallerManager: Parameters ---------- + entry : + The tool's full registry entry. When supplied, artifacts + are routed to the install-root destination declared by + ``filesystem.install`` / the v3.4 router instead of the + flat ``tools_root/``. license_spdx : SPDX ID for the tool's license. If provided AND a ``license_gate`` was passed to the InstallerManager @@ -806,21 +1006,29 @@ class InstallerManager: strategies: dict[str, Any] = { "ollama": lambda: self.install_ollama(pkg, tool_id), - "uv": lambda: self.install_uv(pkg, tool_id, env_overrides), - "pipx": lambda: self.install_pipx(pkg, tool_id, env_overrides), - "pip": lambda: self.install_pip(pkg, tool_id, env_overrides), + "uv": lambda: self.install_uv( + pkg, tool_id, env_overrides, entry, + ), + "pipx": lambda: self.install_pipx( + pkg, tool_id, env_overrides, entry, + ), + "pip": lambda: self.install_pip( + pkg, tool_id, env_overrides, entry, + ), "script": lambda: self.install_script( cmd, ctx, tool_id, env_overrides, ), "pacman": lambda: self.install_pacman(pkg), "dnf": lambda: self.install_dnf(pkg), "apt": lambda: self.install_apt(pkg), - "npm": lambda: self.install_npm(pkg, tool_id, env_overrides), + "npm": lambda: self.install_npm( + pkg, tool_id, env_overrides, entry, + ), "git": lambda: self.install_git( - pkg, tool_id, post_install, env_overrides, + pkg, tool_id, post_install, env_overrides, entry, ), "git_node": lambda: self.install_git_node( - pkg, tool_id, post_install, + pkg, tool_id, post_install, entry, ), "custom": lambda: self.install_custom(pkg, tool_id), } @@ -852,6 +1060,7 @@ class InstallerManager: inst_type=meta.get("installer", {}).get("type", "pacman"), pkg=meta.get("installer", {}).get("pkg", ""), cmd=meta.get("installer", {}).get("cmd", ""), + entry=meta, ) for tid, meta in tools.items() } @@ -867,15 +1076,28 @@ class InstallerManager: post_install: str | None = None, env_overrides: dict[str, str] | None = None, license_spdx: str | None = None, + entry: dict[str, Any] | None = None, + update_if_found: bool = True, ) -> str: """Install a tool with preflight detection. - If the tool is already installed and *force* is False, returns - a message saying the tool exists and suggesting an update. + If the tool is already installed and *force* is False: + + * ``update_if_found=True`` (default) — run the update path + (:meth:`update_tool`), honouring the registry's + ``installer.update_cmd`` when declared. This is what makes + the UI's single Deploy/Update button behave as a smooth + in-place updater. + * ``update_if_found=False`` — return the historical + "already installed" notice without touching anything. + If *force* is True, proceeds with installation regardless. Parameters ---------- + entry : + The tool's full registry entry (routes artifacts + enables + update_cmd). license_spdx : SPDX ID for the tool's license. Forwarded to ``run()`` for gate checking. @@ -884,16 +1106,152 @@ class InstallerManager: # subprocess call on a blocked tool. self._check_license(tool_id, license_spdx) - check = self.preflight(tool_id, inst_type, pkg, cmd) + check = self.preflight(tool_id, inst_type, pkg, cmd, entry) if check["found"] and not force: - return ( - f"Tool '{tool_id}' already installed at {check['location']}. " - f"Version: {check['version'] or 'unknown'}. " - f"Use force=True to update." + if not update_if_found: + return ( + f"Tool '{tool_id}' already installed at " + f"{check['location']}. " + f"Version: {check['version'] or 'unknown'}. " + f"Use force=True to update." + ) + update_cmd = (entry or {}).get("installer", {}).get( + "update_cmd", "", + ) if entry else "" + return self.update_tool( + tool_id=tool_id, + inst_type=inst_type, + pkg=pkg, + cmd=cmd, + update_cmd=update_cmd or "", + entry=entry, + env_overrides=env_overrides, + post_install=post_install, + detected=check, ) return self.run( inst_type, pkg, cmd, ctx, tool_id, - post_install, env_overrides, + post_install, env_overrides, license_spdx, entry, + ) + + # ── Updates ───────────────────────────────────────────────────── + + def update_tool( + self, + tool_id: str, + inst_type: str, + pkg: str, + cmd: str = "", + update_cmd: str = "", + entry: dict[str, Any] | None = None, + env_overrides: dict[str, str] | None = None, + post_install: str | None = None, + detected: dict[str, Any] | None = None, + ) -> str: + """Update an already-installed tool in place. + + Resolution order: + + 1. **Registry ``update_cmd``** — rendered with the path-tree + placeholders plus ``{install_dir}`` and executed in the + tool's install directory (e.g. LibreChat's + ``git pull --ff-only && yarn install && yarn build``). + 2. **Per-type defaults** — ``uv tool upgrade``, ``pipx + upgrade``, per-tool venv ``pip install -U``, ``git pull + --ff-only`` (+ node rebuild for ``git_node``), ``npm + update``. + 3. **System packages** (pacman / dnf / apt) and ``custom`` + installs return guidance instead of shelling out. + + The working directory is the *detected* install location when + known, else the router destination — so updates land where the + tool actually lives, never forking a second copy. + """ + _validate_tool_id(tool_id) + dest = ( + detected.get("location") + if detected and detected.get("location") + else self._existing_dir_for(tool_id, entry) + ) + env = self._env(tool_id, env_overrides) + + # 1. Registry-declared update command wins. + if update_cmd: + rendered = self._render_cmd(update_cmd, dest) + logger.info("Updating %s via update_cmd: %s", tool_id, rendered) + subprocess.run( + ["bash", "-c", rendered], check=True, env=env, + cwd=dest if os.path.isdir(dest) else None, + timeout=1800, + ) + return f"Update completed for {tool_id} (update_cmd) in {dest}." + + # 2. Per-type defaults. + pkg_first = shlex.split(pkg)[0] if pkg else pkg + if inst_type == "uv": + try: + subprocess.run( + ["uv", "tool", "upgrade", pkg_first], + check=True, env=env, timeout=600, + ) + return f"UV tool '{pkg_first}' upgraded." + except subprocess.CalledProcessError: + # upgrade failed (e.g. broken env) — reinstall keeps + # --with extras alive via install_uv's step-down. + logger.info( + "uv upgrade failed for %s; reinstalling", pkg_first, + ) + return self.install_uv( + pkg, tool_id, env_overrides, entry, + ) + if inst_type == "pipx": + try: + subprocess.run( + ["pipx", "upgrade", pkg_first], + check=True, env=env, timeout=600, + ) + return f"pipx '{pkg_first}' upgraded." + except subprocess.CalledProcessError: + logger.info( + "pipx upgrade failed for %s; reinstalling", pkg_first, + ) + return self.install_pipx( + pkg, tool_id, env_overrides, entry, + ) + if inst_type == "pip": + venv_pip = os.path.join(dest, ".venv", "bin", "pip") + if os.path.isfile(venv_pip): + _validate_pkg(pkg_first) + subprocess.run( + [venv_pip, "install", "--upgrade", pkg_first], + check=True, env=env, timeout=600, + ) + return f"pip '{pkg_first}' upgraded in {dest}/.venv." + return ( + f"No managed venv for {tool_id}; re-run with force to " + f"reinstall." + ) + if inst_type in ("git", "git_node"): + # install_git / install_git_node already implement + # pull-ff-only-with-fallback + node rebuild semantics. + handler = (self.install_git_node if inst_type == "git_node" + else self.install_git) + msg = handler( + pkg, tool_id, post_install, env_overrides, entry, + ) + return f"Update: {msg}" + if inst_type == "npm": + _validate_pkg(pkg_first) + subprocess.run( + ["npm", "update", "--prefix", dest, pkg_first], + check=True, env=env, timeout=600, + ) + return f"NPM '{pkg_first}' updated in {dest}." + + # 3. Types we deliberately don't auto-update. + return ( + f"'{tool_id}' is a {inst_type}-managed install — update it " + f"with the system package manager or its upstream installer." ) # ── Installation verification ─────────────────────────────────── @@ -905,6 +1263,7 @@ class InstallerManager: pkg: str, cmd: str = "", filesystem: dict[str, str] | None = None, + entry: dict[str, Any] | None = None, ) -> dict[str, Any]: """Run a compliance checklist against a single tool installation. @@ -927,9 +1286,10 @@ class InstallerManager: fs = filesystem or {} tool_dir = os.path.join(self.tools_root, tool_id) - # 1. Native install detected + # 1. Native install detected (multi-root: router + dashboards + # + tools, so pre-v3.4 installs still verify) location, version = self._detect_installation( - tool_id, inst_type, pkg, cmd, + tool_id, inst_type, pkg, cmd, entry, ) checks.append(VerifyCheck( name="Native Install", diff --git a/src/ai_lsc/stack/export.py b/src/ai_lsc/stack/export.py index 0564428..241f3e4 100755 --- a/src/ai_lsc/stack/export.py +++ b/src/ai_lsc/stack/export.py @@ -6,34 +6,59 @@ Contains pure-logic functions for: * **build_stack_spec** -- serialises the current pipeline state plus registry metadata into a portable JSON spec. * **ContainerBackend** -- generates Podman/Docker compose YAML, - LXC container configs, Firecracker microVM configs, or JSON fallback - from that spec. + LXC container configs, Firecracker microVM configs, QCOW2/QCOW VM + disk images, QCrows Sovereign Bundle (.svb) archives, or JSON + fallback from that spec. No UI code here. All path operations use :mod:`pathlib`. """ from __future__ import annotations +import gzip +import hashlib +import io import json -from datetime import datetime +import os +import platform +import re +import shutil +import subprocess +import tarfile +from datetime import datetime, timezone from pathlib import Path from typing import Any -from ai_lsc.constants import BASE_DIR, STACK_SCHEMA_VERSION +from ai_lsc.constants import APP_VERSION, BASE_DIR, STACK_SCHEMA_VERSION from ai_lsc.registry.manager import RegistryManager +from ai_lsc.utils.logging import get_logger from ai_lsc.utils.paths import build_path_tree +# ── QCrows format constants ─────────────────────────────────────────── +# +# QCrows (cue-crows) is the in-house self-describing VM container image +# format for Kata Containers, defined in the cockpit-kata project +# (qcrows-spec.md + the qcrows-pack / qcrows-verify / qcrows-inspect / +# qcrows-export tools — the master implementation). A .qcrows archive +# bundles metadata.toml + menu.toml + hashes.sha256 + rootfs.tar.* + +# initrd + kernel/ (kernel REQUIRED in v0.2+) + optional provenance. +QCROWS_SPEC_VERSION = "0.2.0" + +# ── Sovereign Bundle (.svb) format constants ───────────────────────── +# +# The sorcery-go project's single-file bundle format +# (pkg/cauldron/cauldron.go::BundleSovereign): tar.gz whose first entry +# is a METADATA.json manifest (format id +# "sorcery-sovereign-bundle-v1"), optionally a raw 64-byte ed25519 +# SIGNATURE.sig, a per-file MANIFEST.txt with sha256 hashes, then the +# payload entries. Unsigned bundles are legal per the spec. +SVB_FORMAT_ID = "sorcery-sovereign-bundle-v1" + # H-17: single helper for the placeholder-resolution chain that was # previously duplicated in three methods (compose / lxc / firecracker). -_PLACEHOLDER_KEYS = ( - "base_dir", - "tools_root", - "models_root", - "workspaces_root", -) - - +# Since v3.4 every path-tree key resolves (dashboards_root, runtime_root, +# configs_root, …), not just the original four. def _resolve_placeholders(cmd: str, paths: dict[str, Any]) -> str: """Replace ``{base_dir}`` / ``{tools_root}`` / ... placeholders in *cmd*. @@ -42,8 +67,8 @@ def _resolve_placeholders(cmd: str, paths: dict[str, Any]) -> str: if not cmd: return "" resolved = cmd - for key in _PLACEHOLDER_KEYS: - resolved = resolved.replace("{" + key + "}", str(paths.get(key, ""))) + for key, value in paths.items(): + resolved = resolved.replace("{" + key + "}", str(value)) return resolved @@ -549,6 +574,954 @@ class ContainerBackend: return launch_script + # ── VM-disk image backend (qcow2 / qcow / raw, extensible) ──────── + + # Formats this backend can produce today, all via the rootless + # mkfs.ext4 -d + qemu-img pipeline in write_vm_image(). Each + # format writes to exports//. New formats — notably the + # in-house "qcrows" image format whose reference is pending — + # register here (and in write()'s dispatch) with their own builder; + # everything else (staging, manifests, UI listing) is shared. + VM_IMAGE_FORMATS: tuple[str, ...] = ("qcow2", "qcow", "raw") + # Kept as an alias for pre-v3.4 callers. + QCOW_FORMATS = VM_IMAGE_FORMATS + _QCOW_DEFAULT_SIZE_MB = 2048 + _QCOW_MIN_FREE_MB = 64 + + @staticmethod + def _require_binary(name: str, hint: str) -> str: + """Return the resolved path of *name* or raise with an install hint.""" + path = shutil.which(name) + if not path: + raise FileNotFoundError( + f"{name!r} not found on PATH — required for VM disk " + f"exports. Install it with: {hint}" + ) + return path + + def _stage_rootfs( + self, + spec: dict, + staging: Path, + compose_path: Path, + ) -> None: + """Populate the *staging* rootfs directory for a qcow export. + + Layout inside the image:: + + /opt/ai-lsc/stack.json -- the portable stack spec + /opt/ai-lsc/docker-compose.yml -- compose file for the stack + /root/start.sh -- boot script (compose, or + direct launch fallback) + /etc/ai-lsc-release -- image metadata + """ + pkg_dir = staging / "opt" / "ai-lsc" + pkg_dir.mkdir(parents=True, exist_ok=True) + + (pkg_dir / "stack.json").write_text( + json.dumps(spec, indent=2), encoding="utf-8", + ) + shutil.copy2(compose_path, pkg_dir / "docker-compose.yml") + + (staging / "etc").mkdir(parents=True, exist_ok=True) + # `created` comes from the spec, not wall clock — staging must + # be a pure function of the spec so identical specs hash + # identically (payload_sha determinism for .svb bundles). + (staging / "etc" / "ai-lsc-release").write_text( + "\n".join([ + "name=ai-lsc-stack", + f"created={spec.get('created', datetime.now().isoformat())}", + f"schema={spec.get('schema', STACK_SCHEMA_VERSION)}", + f"tools={len(spec.get('tools', []))}", + f"tool_ids={','.join(t['id'] for t in spec.get('tools', []))}", + ]) + "\n", + encoding="utf-8", + ) + + root_dir = staging / "root" + root_dir.mkdir(parents=True, exist_ok=True) + start_sh = root_dir / "start.sh" + start_sh.write_text( + "\n".join([ + "#!/usr/bin/env bash", + "# AI-LSC VM-image stack boot script.", + "# Prefers container bring-up via podman/docker compose;", + "# falls back to direct tmux launches from stack.json.", + "set -euo pipefail", + 'STACK_DIR="/opt/ai-lsc"', + "", + "if command -v podman-compose >/dev/null 2>&1; then", + ' exec podman-compose -f "$STACK_DIR/docker-compose.yml" up -d', + "elif command -v docker >/dev/null 2>&1; then", + ' exec docker compose -f "$STACK_DIR/docker-compose.yml" up -d', + "fi", + "", + 'echo "[ai-lsc] No compose runtime found — direct launch mode."', + 'python3 - "$STACK_DIR/stack.json" <<"PY"', + "import json, shlex, subprocess, sys", + "spec = json.load(open(sys.argv[1]))", + "for tool in spec.get('tools', []):", + " cmd = (tool.get('launcher') or {}).get('cmd', '')", + " if cmd.strip():", + " print('[ai-lsc] launching', tool['id'])", + " subprocess.Popen(['bash', '-c', cmd])", + "PY", + "wait", + ]) + "\n", + encoding="utf-8", + ) + start_sh.chmod(0o755) + + def write_vm_image( + self, + spec: dict, + image_format: str = "qcow2", + size_mb: int | None = None, + ) -> Path: + """Build a VM disk image of the stack (qcow2 / legacy qcow / raw). + + Pipeline (fully rootless — no loop mounts): + + 1. Stage a rootfs directory (stack spec, compose file, boot + script, release metadata) under ``exports//``. + 2. Create a sparse raw image and populate an ext4 filesystem + directly from the staging tree via ``mkfs.ext4 -d``. + 3. Convert to the requested format with ``qemu-img convert`` + (compressed for qcow2/qcow). + + The result is a *rootfs* disk (no bootloader): boot it with an + external kernel — e.g. pair with the Firecracker export's + ``vmlinux`` and ``root=/dev/vda`` — or attach it to any VM and + install a bootloader with virt-customize. + + Writes a ``.manifest.json`` sidecar with the spec, + size, sha256, and ``qemu-img info`` output. + + This method is the shared skeleton for every format in + :attr:`VM_IMAGE_FORMATS`; an in-house builder (e.g. the pending + "qcrows" format) either reuses it or registers alongside it + in :meth:`write` — staging, sidecars, and UI listing are + format-agnostic. + + Returns the path of the written image. + """ + if image_format not in self.VM_IMAGE_FORMATS: + raise ValueError( + f"unsupported VM image format {image_format!r} " + f"(expected one of {self.VM_IMAGE_FORMATS})" + ) + qemu_img = self._require_binary( + "qemu-img", + "pacman -S qemu-img # or: apt install qemu-utils", + ) + mkfs = self._require_binary( + "mkfs.ext4", + "pacman -S e2fsprogs # or: apt install e2fsprogs", + ) + + qcow_dir = self.exports_root / image_format + qcow_dir.mkdir(parents=True, exist_ok=True) + + # Deterministic-but-unique image name (second-precision, with a + # collision guard for repeated exports inside one second). + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + image_path = qcow_dir / f"ai-lsc-stack-{stamp}.{image_format}" + bump = 0 + while image_path.exists(): + bump += 1 + image_path = qcow_dir / ( + f"ai-lsc-stack-{stamp}-{bump}.{image_format}" + ) + + # ── Stage the rootfs ───────────────────────────────────────── + staging = qcow_dir / f".staging-{image_path.stem}" + staging.mkdir(parents=True, exist_ok=True) + raw_path = qcow_dir / f"{image_path.stem}.raw" + try: + compose_file = self.write_compose( + spec, backend_type="podman", + ) + self._stage_rootfs(spec, staging, compose_file) + + # ── Size the raw image (staging usage + headroom) ──────── + used_bytes = sum( + f.stat().st_size + for f in staging.rglob("*") if f.is_file() + ) + needed_mb = ( + used_bytes // (1024 * 1024) * 4 + + self._QCOW_MIN_FREE_MB + ) + total_mb = max( + size_mb or 0, + self._QCOW_DEFAULT_SIZE_MB, + needed_mb, + ) + with open(raw_path, "wb") as fh: + fh.truncate(total_mb * 1024 * 1024) + + # ── Populate ext4 without mounting (mkfs -d) ───────────── + subprocess.run( + [mkfs, "-F", "-q", "-L", "ai-lsc", + "-d", str(staging), str(raw_path)], + check=True, timeout=600, + ) + + # ── Convert to the target format ───────────────────────── + if image_format == "raw": + shutil.move(str(raw_path), image_path) + else: + try: + subprocess.run( + [qemu_img, "convert", "-c", "-O", image_format, + str(raw_path), str(image_path)], + check=True, timeout=1800, + ) + except subprocess.CalledProcessError as exc: + if image_format == "qcow": + raise RuntimeError( + "qemu-img refused to write legacy qcow (v1) — " + "write support was removed in newer qemu " + "releases. Export as 'qcow2' instead." + ) from exc + raise + + # ── Sidecar manifest ───────────────────────────────────── + sha = hashlib.sha256() + with open(image_path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + sha.update(chunk) + info = subprocess.run( + [qemu_img, "info", "--output=json", str(image_path)], + capture_output=True, text=True, check=True, timeout=30, + ).stdout + manifest = { + "image": image_path.name, + "format": image_format, + "virtual_size_mb": total_mb, + "sha256": sha.hexdigest(), + "created": datetime.now().isoformat(), + "stack": { + "schema": spec.get("schema"), + "tools": [t["id"] for t in spec.get("tools", [])], + "ports": spec.get("ports", {}), + }, + "qemu_img_info": json.loads(info), + "boot_hint": ( + "rootfs disk (no bootloader): boot with an external " + "kernel and root=/dev/vda, e.g. the Firecracker " + "export's vmlinux" + ), + } + (Path(str(image_path) + ".manifest.json")).write_text( + json.dumps(manifest, indent=2), encoding="utf-8", + ) + finally: + shutil.rmtree(staging, ignore_errors=True) + raw_path.unlink(missing_ok=True) + + return image_path + + # ── QCrows (.qcrows) VM-container image backend ──────────────────── + # + # Mirrors the cockpit-kata master implementation (qcrows-pack / + # qcrows-spec.md v0.2.0). Conformance details that matter: + # * tar members carry the "./" prefix exactly like qcrows-pack's + # `tar czf - ./*` (qcrows-inspect tries ./metadata.toml first) + # * hashes.sha256 is byte-compatible with `sha256sum -c` + # * the tar uses PAX format per spec §2.1 (qcrows-pack itself + # writes GNU format — spec-strict consumers prefer PAX) + # * kernel/ (binary + config) is REQUIRED in v0.2+ — sourced from + # runtime/qcrows/, the firecracker dir, or the host as fallback + + # Required Kata kernel options (qcrows-verify warns when absent). + _KATA_KERNEL_OPTS = ( + "CONFIG_VSOCKETS", "CONFIG_VIRTIO", "CONFIG_VIRTIO_PCI", + "CONFIG_DEVTMPFS", "CONFIG_DEVTMPFS_MOUNT", + ) + + def write_qcrows(self, spec: dict, compress: bool = False) -> Path: + """Build a QCrows VM container image (.qcrows) of the stack. + + Produces a spec-v0.2.0 archive (see cockpit-kata's + ``qcrows-spec.md`` — the master implementation): + + .. code-block:: text + + (PAX; gzip only if compress=True) + ├── metadata.toml image manifest (generated) + ├── menu.toml Cockpit UI menu entry + ├── hashes.sha256 sha256sum-format integrity list + ├── rootfs.tar.gz staged ai-lsc stack rootfs + ├── boot-params.conf console + root device params + ├── build.toml ai-lsc build provenance + ├── spec.md per-image build guide + └── kernel/ + ├── vmlinuz|vmlinux REQUIRED guest kernel + └── config REQUIRED kernel .config + + Kernel sourcing (first hit wins): + + 1. ``/runtime/qcrows/vmlinuz|vmlinux`` + ``config`` + (the canonical home for a dedicated guest kernel) + 2. ``/containers/firecracker/vmlinux`` (+ ``.config``) + 3. Host kernel fallback — ``/boot/vmlinuz-$(uname -r)`` with + config from ``/boot/config-*``, ``/usr/lib/modules// + config``, or ``/proc/config.gz``. The host kernel is a + stand-in: prefer a purpose-built guest kernel for real Kata + deployments (a warning is logged when the fallback is used). + + Raises ``FileNotFoundError`` with candidate paths when no + kernel can be located (QCrows v0.2 refuses kernel-less images + — same behaviour as qcrows-pack). + + Returns the path of the written image. + """ + out_dir = self.exports_root / "qcrows" + out_dir.mkdir(parents=True, exist_ok=True) + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + image_path = out_dir / f"ai-lsc-stack-{stamp}.qcrows" + bump = 0 + while image_path.exists(): + bump += 1 + image_path = out_dir / f"ai-lsc-stack-{stamp}-{bump}.qcrows" + + arch = self._qcrows_arch() + kernel_src, kernel_cfg, kernel_fmt, kernel_origin = ( + self._qcrows_locate_kernel() + ) + kernel_version = self._qcrows_kernel_version( + kernel_src, kernel_cfg, + ) + config_warnings = self._qcrows_check_kata_opts(kernel_cfg) + + staging = out_dir / f".staging-{image_path.stem}" + staging.mkdir(parents=True, exist_ok=True) + try: + # ── rootfs.tar.gz — the staged stack tree ──────────────── + compose_file = self.write_compose(spec, backend_type="podman") + rootfs_dir = staging / "rootfs-tree" + self._stage_rootfs(spec, rootfs_dir, compose_file) + rootfs_path = staging / "rootfs.tar.gz" + with open(rootfs_path, "wb") as fh: + with gzip.GzipFile(fileobj=fh, mode="wb", mtime=0) as gz: + with tarfile.open( + fileobj=gz, mode="w", format=tarfile.PAX_FORMAT, + ) as tf: + for p in sorted(rootfs_dir.rglob("*")): + arc = "./" + str( + p.relative_to(rootfs_dir), + ).replace(os.sep, "/") + info = tf.gettarinfo(str(p), arcname=arc) + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mtime = 0 + if p.is_file(): + with open(p, "rb") as src: + tf.addfile(info, src) + else: + tf.addfile(info) + shutil.rmtree(rootfs_dir) + + # ── kernel/ (REQUIRED) ──────────────────────────────────── + kdir = staging / "kernel" + kdir.mkdir() + shutil.copy2(kernel_src, kdir / kernel_fmt) + cfg_bytes = self._qcrows_read_config(kernel_cfg) + (kdir / "config").write_bytes(cfg_bytes) + + # ── boot-params.conf ───────────────────────────────────── + (staging / "boot-params.conf").write_text( + "console=ttyS0\nroot=/dev/vda\nrw\n", + encoding="utf-8", + ) + + # ── metadata.toml / menu.toml / build.toml / spec.md ──── + tool_ids = [t["id"] for t in spec.get("tools", [])] + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + (staging / "metadata.toml").write_text(self._qcrows_metadata( + spec=spec, arch=arch, now=now, + kernel_fmt=kernel_fmt, kernel_version=kernel_version, + kernel_size_mb=kernel_src.stat().st_size // (1024 * 1024), + rootfs_size_mb=rootfs_path.stat().st_size // (1024 * 1024), + ), encoding="utf-8") + (staging / "menu.toml").write_text( + self._qcrows_menu(tool_ids), encoding="utf-8", + ) + (staging / "build.toml").write_text(self._qcrows_build( + arch=arch, now=now, kernel_origin=kernel_origin, + ), encoding="utf-8") + (staging / "spec.md").write_text( + self._qcrows_spec_md(spec, kernel_origin), encoding="utf-8", + ) + + # ── hashes.sha256 — sha256sum -c compatible ────────────── + hash_lines = [] + for p in sorted(staging.rglob("*")): + if not p.is_file(): + continue + rel = "./" + str(p.relative_to(staging)).replace(os.sep, "/") + if rel == "./hashes.sha256": + continue + hash_lines.append( + f"{self._sha256_file(p)} {rel}", + ) + (staging / "hashes.sha256").write_text( + "\n".join(hash_lines) + "\n", encoding="utf-8", + ) + + # ── pack the archive (PAX, "./"-prefixed members) ──────── + open_fh = ( + gzip.GzipFile(filename=str(image_path) + ".gz", + mode="wb", mtime=0) + if compress else None + ) + final_path = ( + Path(str(image_path) + ".gz") if compress else image_path + ) + try: + out_stream = open_fh if compress else open( + image_path, "wb", + ) + with out_stream as fh: + with tarfile.open( + fileobj=fh, mode="w", format=tarfile.PAX_FORMAT, + ) as tf: + for p in sorted(staging.rglob("*")): + arc = "./" + str( + p.relative_to(staging), + ).replace(os.sep, "/") + info = tf.gettarinfo(str(p), arcname=arc) + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mtime = 0 + if p.is_file(): + with open(p, "rb") as src: + tf.addfile(info, src) + else: + tf.addfile(info) + finally: + if open_fh is not None: + open_fh.close() + + # ── ai-lsc sidecar ─────────────────────────────────────── + sidecar = { + "image": final_path.name, + "format": "qcrows", + "format_version": QCROWS_SPEC_VERSION, + "sha256": self._sha256_file(final_path), + "created": datetime.now().isoformat(), + "kernel": { + "version": kernel_version, + "format": kernel_fmt, + "origin": kernel_origin, + }, + "config_warnings": config_warnings, + "verify_hint": f"qcrows-verify {final_path}", + "stack": { + "schema": spec.get("schema"), + "tools": tool_ids, + "ports": spec.get("ports", {}), + }, + } + Path(str(final_path) + ".manifest.json").write_text( + json.dumps(sidecar, indent=2), encoding="utf-8", + ) + finally: + shutil.rmtree(staging, ignore_errors=True) + + return final_path + + # ── QCrows helpers ──────────────────────────────────────────────── + + @staticmethod + def _qcrows_arch() -> str: + machine = platform.machine() or "unknown" + return {"amd64": "x86_64", "arm64": "aarch64"}.get(machine, machine) + + def _qcrows_locate_kernel(self) -> tuple[Path, Path, str, str]: + """Find a guest kernel + config. → (kernel, config, fmt, origin).""" + base = build_path_tree(Path(self.exports_root).parent) + + def _cfg_for(kdir: Path) -> Path | None: + for name in ("config", ".config", "config-" + os.uname().release): + cand = kdir / name + if cand.is_file(): + return cand + return None + + # 1. Dedicated guest kernel home + qdir = Path(str(base["runtime_root"])) / "qcrows" + for fmt in ("vmlinuz", "vmlinux"): + k = qdir / fmt + if k.is_file(): + cfg = _cfg_for(qdir) + if cfg: + return k, cfg, fmt, str(k) + # 2. Firecracker convention + fdir = Path(str(base["base_dir"])) / "containers" / "firecracker" + k = fdir / "vmlinux" + if k.is_file(): + cfg = _cfg_for(fdir) + if cfg: + return k, cfg, "vmlinux", str(k) + # 3. Host kernel fallback — exact release name first (distros + # that version the filename), then any /boot/vmlinuz* (Arch + # names it vmlinuz-linux regardless of release). + release = os.uname().release + candidates: list[Path] = [ + Path("/boot") / f"vmlinuz-{release}", + Path("/boot") / f"vmlinux-{release}", + ] + candidates += sorted(Path("/boot").glob("vmlinuz-*")) + candidates += sorted(Path("/boot").glob("vmlinux-*")) + for k in candidates: + if not k.is_file(): + continue + fmt = "vmlinux" if k.name.startswith("vmlinux") else "vmlinuz" + cfgs = [ + Path("/boot") / f"config-{release}", + Path(f"/usr/lib/modules/{release}/config"), + Path("/proc/config.gz"), + ] + for cfg in cfgs: + if cfg.is_file(): + get_logger(__name__).warning( + "qcrows: no dedicated guest kernel found; " + "using HOST kernel %s (drop one at %s for " + "Kata-tuned images)", k, qdir, + ) + return k, cfg, fmt, f"host:{k}" + raise FileNotFoundError( + "qcrows export requires a kernel (QCrows v0.2+). Checked: " + f"{qdir}/vmlinuz|vmlinux+config, {fdir}/vmlinux+config, " + "/boot/vmlinuz- + config. Drop a guest kernel + " + ".config into the runtime/qcrows/ directory and retry." + ) + + @staticmethod + def _qcrows_read_config(cfg: Path) -> bytes: + """Kernel config bytes; decompress /proc/config.gz on the fly.""" + if cfg.name.endswith(".gz"): + with gzip.open(cfg, "rb") as fh: + return fh.read() + return cfg.read_bytes() + + @staticmethod + def _qcrows_kernel_version(kernel: Path, cfg: Path) -> str: + """Mirror qcrows-pack's version probe. + + First printable-string match of ^\\d+\\.\\d+\\.\\d+ in the + kernel binary, else CONFIG_VERSION_SIGNATURE from the config, + else "unknown". + """ + data = kernel.read_bytes() + for m in re.finditer(rb"[\x20-\x7e]{4,}", data): + vm = re.match( + rb"\d+\.\d+\.\d+", m.group(0), + ) + if vm: + return vm.group(0).decode() + try: + for line in ContainerBackend._qcrows_read_config( + cfg, + ).decode("utf-8", errors="ignore").splitlines(): + if line.startswith("CONFIG_VERSION_SIGNATURE="): + vm = re.search(r"\d+\.\d+\.\d+", line) + if vm: + return vm.group(0) + except OSError: + pass + return "unknown" + + def _qcrows_check_kata_opts(self, cfg: Path) -> list[str]: + """Return the list of missing required Kata kernel options.""" + try: + text = self._qcrows_read_config(cfg).decode( + "utf-8", errors="ignore", + ) + except OSError: + return ["kernel config unreadable"] + missing = [ + opt for opt in self._KATA_KERNEL_OPTS + if f"{opt}=y" not in text + ] + if missing: + get_logger(__name__).warning( + "qcrows: kernel config lacks %s (qcrows-verify will " + "warn; a purpose-built guest kernel is recommended " + "for Kata)", ", ".join(missing), + ) + return missing + + @staticmethod + def _qcrows_metadata( + spec: dict, + arch: str, + now: str, + kernel_fmt: str, + kernel_version: str, + kernel_size_mb: int, + rootfs_size_mb: int, + ) -> str: + """metadata.toml — mirrors qcrows-pack's generator (incl. the + section order qcrows-inspect's first-match parsing relies on).""" + tool_ids = [t["id"] for t in spec.get("tools", [])] + name = "ai-lsc-stack" if tool_ids else "ai-lsc-stack-empty" + desc = ( + f"AI-LSC stack image ({len(tool_ids)} tools: " + f"{', '.join(tool_ids[:8])}" + + ("…" if len(tool_ids) > 8 else "") + ")" + ) + return f"""[ qcrows ] +format_version = "{QCROWS_SPEC_VERSION}" + +[ image ] +name = "{name}" +version = "{APP_VERSION}" +description = "{desc}" +arch = "{arch}" +os = "linux" +created_at = "{now}" + +[ image.compatibility ] +hypervisors = [ "qemu", "cloud-hypervisor", "firecracker" ] +kata_runtime_min = "2.5" + +[ image.resources ] +min_vcpus = 1 +min_memory_mb = 256 +recommended_vcpus = 2 +recommended_memory_mb = 1024 + +[ kernel ] +version = "{kernel_version}" +included = true +path = "kernel/{kernel_fmt}" +format = "{kernel_fmt}" +config_path = "kernel/config" +size_mb = {kernel_size_mb} + +[ initrd ] +included = false +type = "" +path = "" + +[ rootfs ] +type = "tar-gzip" +path = "rootfs.tar.gz" +size_mb = {rootfs_size_mb} + +[ agent ] +name = "ai-lsc-stack-runner" +protocol = "vsock" + +[ boot_params ] +included = true +path = "boot-params.conf" +""" + + @staticmethod + def _qcrows_menu(tool_ids: list[str]) -> str: + return f"""[ menu ] +label = "AI-LSC Stack ({len(tool_ids)} tools)" +category = "custom" +icon = "box" +priority = 50 + +[ menu.details ] +init_system = "auto" +package_count = {len(tool_ids)} +shell = "/bin/sh" + +[ menu.tags ] +[ "suitable-for" ] +workloads = [ "ai-stack" ] +environments = [ "development" ] + +[ menu.actions ] +import = true +deploy = false +edit_config = true +""" + + @staticmethod + def _qcrows_build(arch: str, now: str, kernel_origin: str) -> str: + return f"""[ build ] +system = "custom" +timestamp = "{now}" +host_arch = "{platform.machine() or "unknown"}" +target_arch = "{arch}" + +[ build.options ] +generator = "ai-lsc {APP_VERSION}" +stack_manager = "AI-LSC" + +[ build.sources ] +kernel_source = "{kernel_origin}" + +[ build.reproducibility ] +reproducible = true +build_id = "ai-lsc-stack-{now}" +""" + + @staticmethod + def _qcrows_spec_md(spec: dict, kernel_origin: str) -> str: + tools = "\n".join( + f"- `{t['id']}` — {t.get('name', t['id'])}" + for t in spec.get("tools", []) + ) or "- (empty stack)" + return f"""# AI-LSC Stack QCrows Image + +Generated by AI-LSC {APP_VERSION}. This image packages the active +ai-lsc tool stack as a QCrows (v{QCROWS_SPEC_VERSION}) VM container +image for Kata-compatible runtimes. + +## Contents + +{tools} + +## Reproduce + +1. Select the same tools in ai-lsc and export as QCrows. +2. The rootfs carries `/opt/ai-lsc/stack.json`, a compose file and + `/root/start.sh` (compose bring-up with direct-launch fallback). +3. Kernel source: `{kernel_origin}` — supply a Kata-tuned guest kernel + under `/runtime/qcrows/` (vmlinuz|vmlinux + config) to override + the host-kernel fallback. + +## Verify + + qcrows-verify .qcrows + qcrows-inspect .qcrows +""" + + # ── Sovereign Bundle (.svb) backend ─────────────────────────────── + + def write_svb(self, spec: dict) -> Path: + """Build a Sovereign Bundle (.svb) of the stack. + + Implements the ``sorcery-sovereign-bundle-v1`` format defined + in the sorcery-go project (``pkg/cauldron/cauldron.go:: + BundleSovereign``) — a single-file portable artifact: + + .. code-block:: text + + + ├── METADATA.json (SVBMetadata manifest, first entry) + ├── SIGNATURE.sig (raw 64-byte ed25519 — only if signed) + ├── MANIFEST.txt (per-file sha256 listing) + ├── opt/ai-lsc/stack.json + docker-compose.yml + ├── root/start.sh (bundle boot script) + └── etc/ai-lsc-release + + Signing is optional and dependency-free: the bundle is unsigned + unless BOTH the ``cryptography`` package is importable AND a + raw hex ed25519 private key is supplied via the + ``AI_LSC_SVB_SIGNING_KEY`` environment variable. When signed, + the signature covers the sha256 of the payload tar.gz bytes — + matching sorcery-go's builder semantics. + + Note (upstream divergence, documented for the Coven maintainer): + sorcery-go's *verifier* (``VerifySVBSignature``) hashes the + concatenated payload entry contents rather than the inner + tar.gz bytes, so even sorcery-go's own bundles do not satisfy + their verifier's recomputation. This writer follows the + builder; reconcile in sorcery-go before relying on cross-tool + signature checks. + + Writes the bundle to ``exports/svb/`` plus an ai-lsc-style + ``.manifest.json`` sidecar. Returns the path of the + written bundle. + """ + out_dir = self.exports_root / "svb" + out_dir.mkdir(parents=True, exist_ok=True) + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + bundle_path = out_dir / f"ai-lsc-stack-{stamp}.svb" + bump = 0 + while bundle_path.exists(): + bump += 1 + bundle_path = out_dir / f"ai-lsc-stack-{stamp}-{bump}.svb" + + # ── Stage the payload tree (shared with the qcow pipeline) ── + staging = out_dir / f".staging-{bundle_path.stem}" + staging.mkdir(parents=True, exist_ok=True) + try: + compose_file = self.write_compose(spec, backend_type="podman") + self._stage_rootfs(spec, staging, compose_file) + + # Deterministic payload ordering, mirroring the Go builder. + entries: list[tuple[str, Path]] = sorted( + (str(p.relative_to(staging)), p) + for p in staging.rglob("*") if p.is_file() + ) + entry_hashes: dict[str, str] = {} + total_size = 0 + for tar_path, host in entries: + entry_hashes[tar_path] = self._sha256_file(host) + total_size += host.stat().st_size + + # ── Inner payload tar.gz (hashed, matching the builder) ── + inner = io.BytesIO() + # Fixed gzip mtime + zeroed entry mtimes/owners keep + # payload_sha a pure function of payload content — repeated + # exports of an identical staging tree hash identically + # (the Go builder embeds disk mtimes, so its bundles don't). + with gzip.GzipFile(fileobj=inner, mode="wb", mtime=0) as gz: + with tarfile.open(fileobj=gz, mode="w") as tf: + for tar_path, host in entries: + info = tf.gettarinfo(str(host), arcname=tar_path) + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mtime = 0 + with open(host, "rb") as fh: + tf.addfile(info, fh) + payload_sha = hashlib.sha256(inner.getvalue()).hexdigest() + + # ── Optional ed25519 signing ───────────────────────────── + signature_hex, signer_fpr = self._sign_payload(payload_sha) + + metadata = { + "format": SVB_FORMAT_ID, + "arch": platform.machine() or "unknown", + "created_at": datetime.now(timezone.utc).isoformat(), + "essence_ids": [], + "tools": [ + t.get("name") or t.get("id") + for t in spec.get("tools", []) + ], + "payload_sha": payload_sha, + "signature": signature_hex, + "signer_fpr": signer_fpr, + "total_size": total_size, + "file_count": len(entries), + "annotations": { + "generator": f"ai-lsc {APP_VERSION}", + "signer_note": "AI-LSC stack export bundle", + }, + } + meta_bytes = json.dumps(metadata, indent=2).encode("utf-8") + + # MANIFEST.txt: Go format is "%-40s sha256:". + manifest = "\n".join( + f"{tar_path:<40} sha256:{digest}" + for tar_path, digest in entry_hashes.items() + ) + "\n" + + # ── Outer tar.gz: METADATA → SIGNATURE → MANIFEST → payload + with open(bundle_path, "wb") as out_fh: + with gzip.GzipFile(fileobj=out_fh, mode="wb", mtime=0) as gz: + with tarfile.open(fileobj=gz, mode="w") as tf: + self._tar_add_bytes(tf, "METADATA.json", meta_bytes) + if signature_hex: + self._tar_add_bytes( + tf, "SIGNATURE.sig", + bytes.fromhex(signature_hex), mode=0o644, + ) + self._tar_add_bytes( + tf, "MANIFEST.txt", manifest.encode("utf-8"), + ) + for tar_path, host in entries: + info = tf.gettarinfo(str(host), arcname=tar_path) + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mtime = 0 + with open(host, "rb") as fh: + tf.addfile(info, fh) + + # ── ai-lsc sidecar ─────────────────────────────────────── + sha = hashlib.sha256() + with open(bundle_path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + sha.update(chunk) + sidecar = { + "bundle": bundle_path.name, + "format": SVB_FORMAT_ID, + "sha256": sha.hexdigest(), + "signed": bool(signature_hex), + "created": datetime.now().isoformat(), + "stack": { + "schema": spec.get("schema"), + "tools": [t["id"] for t in spec.get("tools", [])], + "ports": spec.get("ports", {}), + }, + } + Path(str(bundle_path) + ".manifest.json").write_text( + json.dumps(sidecar, indent=2), encoding="utf-8", + ) + finally: + shutil.rmtree(staging, ignore_errors=True) + + return bundle_path + + @staticmethod + def _tar_add_bytes( + tf: tarfile.TarFile, + name: str, + data: bytes, + mode: int = 0o644, + ) -> None: + """Append a regular-file entry with fixed metadata (deterministic).""" + info = tarfile.TarInfo(name=name) + info.size = len(data) + info.mode = mode + info.mtime = 0 + info.type = tarfile.REGTYPE + tf.addfile(info, io.BytesIO(data)) + + @staticmethod + def _sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + @staticmethod + def _sign_payload(payload_sha_hex: str) -> tuple[str, str]: + """Sign *payload_sha_hex* with the ed25519 key from the env. + + Returns ``(signature_hex, signer_fpr_hex)`` — both empty when + unsigned (no key, or ``cryptography`` not installed). The + fingerprint is sha256 of the raw public key bytes, matching + sorcery-go's ``SignerFPR`` derivation. + + ``AI_LSC_SVB_SIGNING_KEY`` accepts a 32-byte seed (the form + ``cryptography`` calls the private key) or a 64-byte + seed||public concatenation as exported by some tooling — the + first 32 bytes are the seed in that case. + """ + key_hex = os.environ.get("AI_LSC_SVB_SIGNING_KEY", "").strip() + if not key_hex: + return "", "" + try: + from cryptography.hazmat.primitives.asymmetric import ed25519 + from cryptography.hazmat.primitives.serialization import ( + Encoding, + PublicFormat, + ) + except ImportError: + return "", "" + try: + raw = bytes.fromhex(key_hex) + if len(raw) == 64: + raw = raw[:32] # seed||pub → seed + if len(raw) != 32: + raise ValueError( + "AI_LSC_SVB_SIGNING_KEY must be 64-hex seed or " + "128-hex seed+pub key" + ) + priv = ed25519.Ed25519PrivateKey.from_private_bytes(raw) + sig = priv.sign(bytes.fromhex(payload_sha_hex)) + pub_raw = priv.public_key().public_bytes( + Encoding.Raw, PublicFormat.Raw, + ) + return sig.hex(), hashlib.sha256(pub_raw).hexdigest() + except ValueError as exc: + raise ValueError(f"cannot sign .svb bundle: {exc}") from exc + # ── Unified write (auto-selects backend) ────────────────────────── def write( @@ -563,8 +1536,9 @@ class ContainerBackend: spec : Stack specification dict (from ``build_stack_spec``). backend_type : - One of ``"podman"``, ``"docker"``, ``"lxc"``, or - ``"firecracker"``. + One of ``"podman"``, ``"docker"``, ``"lxc"``, + ``"firecracker"``, a bundle format (``"qcrows"``/``"svb"``), + or a VM-disk format (``"qcow2"``, ``"qcow"``, ``"raw"``). Returns ------- @@ -574,4 +1548,16 @@ class ContainerBackend: return self.write_lxc(spec) if backend_type == "firecracker": return self.write_firecracker(spec) - return self.write_compose(spec, backend_type=backend_type) + if backend_type == "qcrows": + return self.write_qcrows(spec) + if backend_type == "svb": + return self.write_svb(spec) + if backend_type in self.VM_IMAGE_FORMATS: + return self.write_vm_image(spec, image_format=backend_type) + if backend_type in ("podman", "docker"): + return self.write_compose(spec, backend_type=backend_type) + raise ValueError( + f"unknown export backend {backend_type!r} — expected one " + f"of: podman, docker, lxc, firecracker, qcrows, svb, " + f"{', '.join(self.VM_IMAGE_FORMATS)}" + ) diff --git a/src/ai_lsc/ui/main_window.py b/src/ai_lsc/ui/main_window.py index a0790bb..9356716 100755 --- a/src/ai_lsc/ui/main_window.py +++ b/src/ai_lsc/ui/main_window.py @@ -219,6 +219,7 @@ if _HAS_QT: self._setup_environment_hierarchy() self._migrate_legacy_state_files() + self._migrate_registry_dir() self.dtach_bin: str | None = find_binary("dtach-ng", "dtach") # License gate — checks every tool's license before the # installer dispatches to a subprocess. SaaS-blocked @@ -245,7 +246,7 @@ if _HAS_QT: self.txt_base_dir.setReadOnly(True) self.registry_mgr = RegistryManager( - os.path.join(self.base_dir, "registry") + os.path.join(self.base_dir, "configs", "registry") ) self.skill_resolver = ( SkillRuntimeResolver(self.skills_root) @@ -360,6 +361,40 @@ if _HAS_QT: filter(None, [uv_bin, npm_bin]) ) + def _migrate_registry_dir(self) -> None: + """One-time move of the app registry into configs/registry. + + v3.4 aligned the base dir with the canonical file-system + map: the app-internal ecosystem registry (ecosystem.json + + user tweaks) is app state and belongs under ``configs/``. + Pre-v3.4 installs kept it at ``/registry/`` — + move it when the legacy dir exists and the new one does + not; remove the legacy dir when it is left empty. Never + raises. + """ + import shutil + from ai_lsc.utils.paths import legacy_registry_dir + + old = legacy_registry_dir(self.base_dir) + new = os.path.join(self.base_dir, "configs", "registry") + try: + if os.path.isdir(old): + os.makedirs(new, exist_ok=True) + for fname in os.listdir(old): + src = os.path.join(old, fname) + dst = os.path.join(new, fname) + if not os.path.exists(dst): + shutil.move(src, dst) + if not os.listdir(old): + os.rmdir(old) + from ai_lsc.utils.logging import get_logger + get_logger(__name__).info( + "Migrated tool registry to %s (legacy %s " + "removed)", new, old, + ) + except OSError: + pass + # ─────────────────────────────────────────────────────────────── # LEFT NAVIGATION RACK LAYOUT # ─────────────────────────────────────────────────────────────── @@ -964,7 +999,7 @@ if _HAS_QT: layout.setContentsMargins(30, 20, 30, 20) header = QHBoxLayout() - lbl_title = QLabel("AI-LSC v3.1 — Ankh of Jah") + lbl_title = QLabel("AI-LSC v3.3.0 — Ankh of Jah") lbl_title.setFont(QFont("Segoe UI", 16)) header.addWidget(lbl_title) header.addStretch() @@ -1368,11 +1403,14 @@ if _HAS_QT: if handler: handler() - # Drift detection + # Drift detection (multi-root: checks the router dir, + # dashboards/, and tools/ so pre-v3.4 installs are not + # false-flagged) state_path = self._get_active_state_file() drift: list[str] = [] if state_path: try: + from ai_lsc.utils.paths import candidate_install_dirs with open(state_path) as f: state = json.load(f) git_types = {"git", "git_node"} @@ -1383,8 +1421,12 @@ if _HAS_QT: and self.registry_mgr.get_tool(tid) .get("installer", {}) .get("type") in git_types - and not os.path.exists( - os.path.join(self.tools_root, tid) + and not any( + d.is_dir() for d in candidate_install_dirs( + tid, + self.registry_mgr.get_tool(tid), + self.base_dir, + ) ) ] except Exception: @@ -1449,6 +1491,11 @@ if _HAS_QT: label = { "lxc": "LXC configs + launch script", "firecracker": "Firecracker microVM configs + launch script", + "qcow2": "QCOW2 VM disk image", + "qcow": "QCOW (legacy) VM disk image", + "raw": "Raw VM disk image", + "qcrows": "QCrows VM container image (.qcrows)", + "svb": "Sovereign Bundle (.svb)", }.get(backend_type, f"{backend_type.capitalize()} Compose") self.log( f"{label} generated: {out_file}", diff --git a/src/ai_lsc/ui/pages/container_stacks_tab.py b/src/ai_lsc/ui/pages/container_stacks_tab.py index 8c889f7..423872b 100755 --- a/src/ai_lsc/ui/pages/container_stacks_tab.py +++ b/src/ai_lsc/ui/pages/container_stacks_tab.py @@ -1,7 +1,8 @@ """ContainerStacksTab widget — lists exported stack files. Displays available stack snapshots and provides export buttons for -Podman Compose and Docker Compose outputs. +Podman Compose, Docker Compose, LXC, Firecracker, and QCOW2/QCOW VM +disk-image outputs. """ import os @@ -60,6 +61,32 @@ if _HAS_QT: lambda: self.main.finalize_stack_export("firecracker") ) header.addWidget(btn_firecracker) + + btn_qcow2 = QPushButton("Export -> QCOW2 Image") + btn_qcow2.setStyleSheet("background-color: #d35400;") + btn_qcow2.clicked.connect( + lambda: self.main.finalize_stack_export("qcow2") + ) + header.addWidget(btn_qcow2) + + btn_qcrows = QPushButton("Export -> QCrows Image (.qcrows)") + btn_qcrows.setStyleSheet("background-color: #7f8c8d;") + btn_qcrows.clicked.connect( + lambda: self.main.finalize_stack_export("qcrows") + ) + header.addWidget(btn_qcrows) + + btn_svb = QPushButton("Export -> Sovereign .svb") + btn_svb.clicked.connect( + lambda: self.main.finalize_stack_export("svb") + ) + header.addWidget(btn_svb) + + btn_qcow = QPushButton("Export -> QCOW (legacy)") + btn_qcow.clicked.connect( + lambda: self.main.finalize_stack_export("qcow") + ) + header.addWidget(btn_qcow) layout.addLayout(header) self.file_list = QListWidget() @@ -75,12 +102,27 @@ if _HAS_QT: # the LXC and Firecracker backends. if fname.endswith((".yml", ".yaml", ".json", ".sh")): self.file_list.addItem(fname) - # Also surface subdirectories that hold per-container / - # per-VM configs (lxc/, firecracker/). + # Image-format dirs (qcow2/, and any future format like the + # in-house qcrows/) list their images + sidecar manifests + # with sizes; config dirs (lxc/, firecracker/) get a plain + # directory entry. for fname in sorted(os.listdir(self.main.exports_root)): + if fname.startswith("."): + # transient .staging-* dirs from in-flight exports + continue fpath = os.path.join(self.main.exports_root, fname) - if os.path.isdir(fpath): + if not os.path.isdir(fpath): + continue + if fname in ("lxc", "firecracker"): self.file_list.addItem(f"{fname}/ (directory)") + continue + for sub in sorted(os.listdir(fpath)): + sub_path = os.path.join(fpath, sub) + if os.path.isfile(sub_path) and not sub.startswith("."): + size_mb = os.path.getsize(sub_path) // (1024 * 1024) + self.file_list.addItem( + f"{fname}/{sub} ({size_mb} MiB)" + ) else: ContainerStacksTab = None diff --git a/src/ai_lsc/ui/pages/git_worktree_tab.py b/src/ai_lsc/ui/pages/git_worktree_tab.py index 7320e1f..22aaec4 100755 --- a/src/ai_lsc/ui/pages/git_worktree_tab.py +++ b/src/ai_lsc/ui/pages/git_worktree_tab.py @@ -612,13 +612,17 @@ if _HAS_QT: mgr = InstallerManager( tools_root=self.main.tools_root, base_bin_dir=self.main.base_bin_dir, + base_dir=self.main.base_dir, ) try: if inst_type == "git_node": - msg = mgr.install_git_node(pkg, tool_id, post_install) + msg = mgr.install_git_node( + pkg, tool_id, post_install, entry=meta, + ) else: msg = mgr.install_git( pkg, tool_id, post_install, env_overrides or None, + entry=meta, ) self.main.log(msg, "GitRepo") self.refresh() diff --git a/src/ai_lsc/ui/pages/service_row.py b/src/ai_lsc/ui/pages/service_row.py index 5642ce4..4e3a256 100755 --- a/src/ai_lsc/ui/pages/service_row.py +++ b/src/ai_lsc/ui/pages/service_row.py @@ -4,7 +4,11 @@ Renders one tool (or skill:-prefixed behavior binding) inside the Tools/Services page. Each row shows the service name, live status, CPU load, port input, model selector (for engine/LLM-runtime services), Ollama pull controls, launcher buttons (CLI/GUI/Web), and -Install/Sync + Start/Stop action buttons. +Install/Update + Start/Stop action buttons. The install button runs +preflight first: missing tools install to their map-aligned root +(dashboards/ for web UIs, tools/ otherwise); already-installed tools +update in place (registry ``update_cmd`` when declared, per-type +default otherwise). All process management is delegated to :class:`~ai_lsc.runtime.executor.RuntimeExecutor` -- this widget @@ -222,7 +226,7 @@ if _HAS_QT: btn_api.clicked.connect(self._open_api_dialog) layout.addWidget(btn_api) - self.btn_update = QPushButton("Install / Sync") + self.btn_update = QPushButton("Install / Update") self.btn_update.setStyleSheet("background-color: #8e44ad;") self.btn_update.clicked.connect(self.smart_install) self._install_btn_original_text = self.btn_update.text() @@ -331,6 +335,7 @@ if _HAS_QT: tool_id=self.tool_id, ctx=ctx, license_spdx=license_spdx, + registry_entry=self.meta, ) QTimer.singleShot( 0, lambda d=desc, t=self.tool_id: self._on_install_done(d, t) diff --git a/src/ai_lsc/ui/pages/verification_tab.py b/src/ai_lsc/ui/pages/verification_tab.py index c2e8bb3..a84d8b5 100755 --- a/src/ai_lsc/ui/pages/verification_tab.py +++ b/src/ai_lsc/ui/pages/verification_tab.py @@ -85,6 +85,7 @@ class VerificationWorker(QThread if _HAS_QT else object): # type: ignore[misc] pkg=installer.get("pkg", ""), cmd=installer.get("cmd", ""), filesystem=fs, + entry=meta, ) self.tool_done.emit(tool_id, result) completed += 1 diff --git a/src/ai_lsc/utils/paths.py b/src/ai_lsc/utils/paths.py index 68ff70f..41f3b31 100755 --- a/src/ai_lsc/utils/paths.py +++ b/src/ai_lsc/utils/paths.py @@ -4,14 +4,26 @@ AI-LSC — Centralised path definitions. Every path in the application is derived from ``BASE_DIR`` using :mod:`pathlib`. Import these in UI and orchestration code instead of constructing paths with ``os.path.join`` ad-hoc. + +This module also owns the **install-root router** — the mapping that +decides where a tool's artifacts live under the canonical /mnt/AI +layout (``tools/`` for standalone compiles, ``dashboards/`` for web +UIs, ``runtime/`` for native binaries) — plus the multi-root +candidate list used for detection of pre-existing installs. """ from __future__ import annotations +import re from pathlib import Path +from typing import Any from ai_lsc.constants import BASE_DIR, REQUIRED_DIRS +# Only path-tree keys are auto-resolvable as launcher placeholders. +# `{port}` / `{model_arg}` are runtime values resolved separately. +_TOOL_ID_RE = re.compile(r"^[A-Za-z0-9_.:\-]+$") + def build_path_tree(base_dir: str | Path | None = None) -> dict[str, Path]: """Return a dict of well-known absolute paths under *base_dir*. @@ -40,20 +52,22 @@ def build_path_tree(base_dir: str | Path | None = None) -> dict[str, Path]: "corpus_root": Path("/mnt/AI/corpus"), # parent of hot/ and cold/ "datasets_root": Path("/mnt/AI/datasets"), # parent of wordlists/, huggingface/, github/ "pipelines_root": Path("/mnt/AI/pipelines"), # ETL / chunking / routing scripts - "configs_root": Path("/mnt/AI/configs"), # app state + templated configs - "registry_root": Path("/mnt/AI/registry"), # app-internal: ecosystem.json + manifests/ + "configs_root": Path("/mnt/AI/configs"), # app state + templated configs + "registry_root": Path("/mnt/AI/configs/registry"), # app-internal: ecosystem.json "agents_root": Path("/mnt/AI/agents"), # configs and chains for autonomous actors "skills_root": Path("/mnt/AI/skills"), # 3rd-party integrations and tool wrappers "projects_root": Path("/mnt/AI/projects"), # parent of active/, labs/, vault/ "blueprints_root": Path("/mnt/AI/blueprints"), # Dockerfiles / build contexts for Podman exports "workspaces_root": Path("/mnt/AI/workspaces"), # Jupyter, OpenNotebook, etc. - "dashboards_root": Path("/mnt/AI/dashboards"), # web UIs (Dashy, Open-WebUI, Hermes WebUI, etc.) - "exports_root": Path("/mnt/AI/exports"), # parent of oci-images/ + "dashboards_root": Path("/mnt/AI/dashboards"), # web UIs (Dashy, Homepage, Open-WebUI, etc.) + "exports_root": Path("/mnt/AI/exports"), # parent of oci-images/, qcow2/, qcrows/ + "exports_qcow2": Path("/mnt/AI/exports/qcow2"), # VM disk images (qcow2 / legacy qcow) + "exports_qcrows": Path("/mnt/AI/exports/qcrows"),# QCrows VM container images (.qcrows) "scripts_root": Path("/mnt/AI/scripts"), # system admin / maintenance automation "logs_root": Path("/mnt/AI/logs"), "backends_root": Path("/mnt/AI/backends"), # S3/MinIO/Ceph connection profiles "distfiles_root": Path("/mnt/AI/distfiles"), # permanent local mirror of source tarballs - "config_root": Path("/mnt/AI/configs"), # app state + templated configs + "config_root": Path("/mnt/AI/configs"), # alias of configs_root } """ root = Path(base_dir) if base_dir is not None else Path(BASE_DIR) @@ -67,7 +81,11 @@ def build_path_tree(base_dir: str | Path | None = None) -> dict[str, Path]: "corpus_root": root / "corpus", "datasets_root": root / "datasets", "pipelines_root": root / "pipelines", - "registry_root": root / "registry", + # App-internal registry state (ecosystem.json + user tweaks) + # lives under configs/ per the v3.4 file-system map. Pre-v3.4 + # installs used /registry — main_window migrates that + # directory on startup. + "registry_root": root / "configs" / "registry", "agents_root": root / "agents", "skills_root": root / "skills", "projects_root": root / "projects", @@ -75,6 +93,8 @@ def build_path_tree(base_dir: str | Path | None = None) -> dict[str, Path]: "workspaces_root": root / "workspaces", "dashboards_root": root / "dashboards", "exports_root": root / "exports", + "exports_qcow2": root / "exports" / "qcow2", + "exports_qcrows": root / "exports" / "qcrows", "scripts_root": root / "scripts", "logs_root": root / "logs", "backends_root": root / "backends", @@ -89,6 +109,121 @@ def build_path_tree(base_dir: str | Path | None = None) -> dict[str, Path]: } +# ── Install-root router ───────────────────────────────────────────── +# +# The v3.4 file-system map gives every top-level directory a purpose: +# tools/ — standalone compiles (built from distfiles) +# runtime/ — native compiled binaries (ollama, llama.cpp, MinIO…) +# dashboards/ — web UIs and landing pages (Open-WebUI, LibreChat…) +# +# A tool's declared ``filesystem.install`` (a path relative to the base +# dir) is the single source of truth. When absent, the router picks a +# root by tool kind — but only when the launcher does not already pin +# the tool to ``{tools_root}`` via a ``cd {tools_root}/`` command +# (a pinned launcher would break if the artifacts moved). +INSTALL_ROOT_DEFAULT = "tools" +INSTALL_ROOT_ROUTES: dict[str, str] = { + "web": "dashboards", # flags.has_web → web UI apps +} + + +def install_dir_for( + entry: dict[str, Any] | None, + tool_id: str = "", + base_dir: str | Path | None = None, +) -> Path: + """Return the canonical install directory for a registry *entry*. + + Resolution order: + + 1. ``filesystem.install`` declared on the entry (relative to the + base dir) — honoured verbatim. + 2. ``flags.has_web`` is set AND the launcher does not reference + ``{tools_root}`` → ``/dashboards//``. + 3. Otherwise → ``/tools//`` (the historical + default, still correct for standalone compiles). + + Falls back to ``tools/`` for empty / malformed entries. + """ + root = Path(base_dir) if base_dir is not None else Path(BASE_DIR) + entry = entry or {} + + rel = (entry.get("filesystem") or {}).get("install", "") + if rel: + # Normalise without allowing traversal outside the base dir. + parts = [p for p in Path(rel).parts if p not in (".", "..")] + clean = Path(*parts) if parts else Path("") + if str(clean) not in ("", "."): + return root / clean + + if tool_id and not _TOOL_ID_RE.fullmatch(tool_id): + tool_id = "" + + launcher_cmd = (entry.get("launcher") or {}).get("cmd", "") or "" + pins_tools_root = "{tools_root}" in launcher_cmd + flags = entry.get("flags") or {} + if (tool_id and flags.get("has_web") + and INSTALL_ROOT_ROUTES.get("web") + and not pins_tools_root): + return root / INSTALL_ROOT_ROUTES["web"] / tool_id + return root / INSTALL_ROOT_DEFAULT / (tool_id or "") + + +def candidate_install_dirs( + tool_id: str, + entry: dict[str, Any] | None = None, + base_dir: str | Path | None = None, +) -> list[Path]: + """Return every directory an install of *tool_id* might live in. + + Ordered best-candidate-first: the router's canonical dir, then the + other plausible roots (dashboards/, tools/), de-duplicated. Used + by installer preflight / verification / drift detection so tools + installed before the v3.4 routing change are still detected. + """ + if not tool_id or not _TOOL_ID_RE.fullmatch(tool_id): + return [] + root = Path(base_dir) if base_dir is not None else Path(BASE_DIR) + entry = entry or {} + + cands: list[Path] = [install_dir_for(entry, tool_id, root)] + for dirname in (INSTALL_ROOT_ROUTES.get("web", "dashboards"), + INSTALL_ROOT_DEFAULT): + cands.append(root / dirname / tool_id) + # Pre-v3.4 git tools could land in a dir named after the repo + # (not the tool_id) — cover the common LibreChat-style casing too. + repo_hint = (entry.get("installer", {}) or {}).get("pkg", "") + if isinstance(repo_hint, str) and repo_hint.rstrip("/").endswith(".git"): + repo_name = repo_hint.rstrip("/").rsplit("/", 1)[-1][:-4] + if repo_name and _TOOL_ID_RE.fullmatch(repo_name): + cands.append(root / INSTALL_ROOT_DEFAULT / repo_name) + + seen: set[Path] = set() + ordered: list[Path] = [] + for c in cands: + if c not in seen: + seen.add(c) + ordered.append(c) + return ordered + + +def legacy_registry_dir(base_dir: str | Path | None = None) -> Path: + """Pre-v3.4 location of the app-internal registry dir.""" + root = Path(base_dir) if base_dir is not None else Path(BASE_DIR) + return root / "registry" + + +def ensure_required_dirs(base_dir: str | Path | None = None) -> list[Path]: + """Create every REQUIRED_DIRS entry under *base_dir*; return the list.""" + root = Path(base_dir) if base_dir is not None else Path(BASE_DIR) + created: list[Path] = [] + for rel in REQUIRED_DIRS: + p = root / rel + p.mkdir(parents=True, exist_ok=True) + created.append(p) + return created + + def resolve_launcher_cmd( cmd_template: str, base_dir: str | Path | None = None, @@ -97,17 +232,30 @@ def resolve_launcher_cmd( ) -> str: """Resolve ``{placeholder}`` tokens in a launcher command template. - Recognised placeholders (case-sensitive): - ``{port}``, ``{base_dir}``, ``{tools_root}``, - ``{models_root}``, ``{workspaces_root}``, ``{model_arg}`` + Every key of :func:`build_path_tree` is recognised as a + placeholder (``{base_dir}``, ``{tools_root}``, ``{dashboards_root}``, + ``{runtime_root}``, ``{configs_root}``, …), plus the runtime values + ``{port}`` and ``{model_arg}``. Unknown placeholders are left + untouched so callers can chain their own substitution. """ paths = build_path_tree(base_dir) + resolved = cmd_template + for key, value in paths.items(): + resolved = resolved.replace("{" + key + "}", str(value)) return ( - cmd_template + resolved .replace("{port}", str(port or "")) - .replace("{base_dir}", str(paths["base_dir"])) - .replace("{tools_root}", str(paths["tools_root"])) - .replace("{models_root}", str(paths["models_root"])) - .replace("{workspaces_root}", str(paths["workspaces_root"])) .replace("{model_arg}", model_arg) ) + + +__all__ = [ + "build_path_tree", + "install_dir_for", + "candidate_install_dirs", + "legacy_registry_dir", + "ensure_required_dirs", + "resolve_launcher_cmd", + "INSTALL_ROOT_DEFAULT", + "INSTALL_ROOT_ROUTES", +]