From 34d429345d79f55e80771b737d2ffaa33485e5d9 Mon Sep 17 00:00:00 2001 From: Jeremy Anderson Date: Sun, 30 Aug 2026 13:58:23 -0400 Subject: [PATCH] AI-LSC v3.3.0 - MoE QA pass 2 --- CHANGES.md | 29 +- QA-PASS-REPORT.md | 60 +++- TODO.md | 11 +- apply_taxonomy_migration.py | 300 ------------------ src/ai_lsc/registry/defaults.py | 2 +- src/ai_lsc/registry/layers/user_interfaces.py | 4 +- src/ai_lsc/stack/connections.py | 8 +- 7 files changed, 88 insertions(+), 326 deletions(-) delete mode 100755 apply_taxonomy_migration.py diff --git a/CHANGES.md b/CHANGES.md index d84d497..e2343de 100755 --- a/CHANGES.md +++ b/CHANGES.md @@ -192,8 +192,7 @@ become L9; IaC and cluster automation land in L10. policy markers are untouched (see whatremains.txt, C-05). - 11 modular layer files realigned: 185 tools migrated to the 10-layer taxonomy (107 shared tools take their structural fields from the - master registry; the other 78 classified explicitly — see - `scripts/apply_10layer_taxonomy.py` for the classification table). + master registry; the other 78 classified explicitly). - `kanban` reconciled: it existed only in `defaults.py` (the canonical source is the layer files), so it is now declared in `layers/user_interfaces.py` as well (L10, Sprints Manager). Merged @@ -238,10 +237,11 @@ become L9; IaC and cluster automation land in L10. - README layer table + tool counts updated to the 10-layer taxonomy (186 tools); quickstart's "13-layer architecture" reference fixed. - Layer-file module docstrings note the 10-layer realignment. -- `scripts/apply_10layer_taxonomy.py` added (the corrected, completed - migration utility — supersedes the root `apply_taxonomy_migration.py`, - whose regexes could not match multi-word layer names and whose - defaults handling was lossy; kept for history). +- One-shot migration utilities (`apply_taxonomy_migration.py`, + `scripts/apply_10layer_taxonomy.py`, and the three + `scripts/backfill_*.py` license/flag backfills) retired post-apply + in the MoE QA pass — they were idempotent and would have left dead + code sitting in the tree. ### Verification @@ -524,6 +524,11 @@ root regardless of where the tarball is extracted: - `scripts/backfill_layer_flags.py` - `scripts/backfill_tool_licenses.py` +> **Note (MoE QA pass):** all three one-shot backfill scripts have since +> been retired post-apply — they were idempotent migrations and leaving +> them in the tree amounted to dead code. The fix history above is kept +> for traceability. + ### Verification (run against this build) ``` @@ -669,7 +674,7 @@ Files: `src/ai_lsc/ui/widgets/workspace_tab.py` (new, ~310 lines), `src/ai_lsc/u - **H-12** LXC `attach_exec` destroys quoting — `command.split()` replaced with `shlex.split(command)`. - **H-13** Redis lock silently bypassed when down — `agents/redis_bridge.py:acquire_lock` and `release_lock` now log a WARNING when Redis is unreachable so operators know concurrent agents could race. - **H-14** `_enforce_quality` false-positive error detection — `agents/orchestrator.py` now uses a word-boundary regex with a negative lookahead `(?![\w\-])` so hyphenated compounds like `error-correction module initialized` are not flagged. -- **H-15** Registry layer files have incomplete flag schemas — `registry/validator.py` now enforces the full 8-key flags schema (`has_cli`, `has_gui`, `has_web`, `is_ollama`, `is_docker`, `is_passive`, `is_mcp`, `is_skills_collection`). `scripts/backfill_layer_flags.py` backfilled 123 flag blocks across all 13 layer files. +- **H-15** Registry layer files have incomplete flag schemas — `registry/validator.py` now enforces the full 8-key flags schema (`has_cli`, `has_gui`, `has_web`, `is_ollama`, `is_docker`, `is_passive`, `is_mcp`, `is_skills_collection`). A one-shot backfill (now retired) populated 123 flag blocks across all 13 layer files. - **H-16** `_inject_skill_stub` returns fake success — `agents/dispatcher.py` now validates the skill exists before reporting success. - **H-17** Placeholder resolution triple-copy in `export.py` — new `_resolve_placeholders()` helper replaces three duplicated 6-line `.replace()` chains in `generate_compose_yaml`, `generate_lxc_configs`, and `generate_firecracker_configs`. - **H-18** Ollama `/api/tools` endpoint does not exist — `agents/ollama_tools.py` `register_all` and `register_single` are now gated behind `_registration_supported = False` with a clear warning. Tool schemas are passed inline to `/api/chat` instead. @@ -792,9 +797,9 @@ After the SaaS blocklist landed, the user asked for a layered license-acceptance - Unknown SPDX ID (not in the license catalog) → error - The `_REQUIRED_FIELDS` set now includes `"license"` - Two backfill scripts added: - - `scripts/backfill_tool_licenses.py` — adds the `license` field to every tool based on a curated override table (85 tools mapped to known licenses: ollama→MIT, vllm→Apache-2.0, grafana→AGPL-3.0, redis→RSALv2, terraform→BSL-1.1, n8n→Sustainable-Use, claude_code→Anthropic-ToS, etc.) - - `scripts/backfill_default_licenses.py` — fills any remaining tools with `"Proprietary"` as the defensive default (83 tools defaulted — the user can review and update these to their actual licenses later) + Two backfill scripts (now retired) populated the initial values: + - One pass added the `license` field based on a curated override table (85 tools mapped to known licenses: ollama→MIT, vllm→Apache-2.0, grafana→AGPL-3.0, redis→RSALv2, terraform→BSL-1.1, n8n→Sustainable-Use, claude_code→Anthropic-ToS, etc.) + - A second pass filled any remaining tools with `"Proprietary"` as the defensive default (83 tools defaulted — the user can review and update these to their actual licenses later) 5. **License gate wired into the installer** — `InstallerManager.__init__` now accepts a `license_gate` parameter. `InstallerManager.run()` and `install_with_preflight()` call `self._check_license(tool_id, license_spdx)` before any subprocess dispatch. If the gate raises `LicenseBlocked` or `LicenseAcceptanceRequired`, the exception propagates up through `RuntimeExecutor.install_tool()` to the UI. @@ -864,14 +869,14 @@ Three latent bugs were introduced during the initial fix wave and caught by the ### Upgrade notes - If you have a v3.0 `pipeline_state.json`, it will continue to work — the schema is unchanged. -- If you maintain custom layer-file entries, run `python scripts/backfill_layer_flags.py` to backfill the 5 new flag keys (`is_ollama`, `is_docker`, `is_passive`, `is_mcp`, `is_skills_collection`) defaulting to `False`. The validator will reject entries missing these keys. +- If you maintain custom layer-file entries, ensure each entry declares all 5 newer flag keys (`is_ollama`, `is_docker`, `is_passive`, `is_mcp`, `is_skills_collection`) defaulting to `False`. The validator will reject entries missing these keys. (An idempotent backfill script was shipped in v3.1 and retired in the MoE QA pass — its work is baked into the layer files.) - If you have hardcoded `sk-ai-lsc-local` API keys in your environment, set `AI_LSC_LITELLM_KEY` and `AI_LSC_OPENWEBUI_KEY` env vars instead — the source-code default is now an empty string. - If you launch the app from a non-default working directory, config is now resolved against `BASE_DIR` instead of `os.getcwd()` — this may move your `config.json` to a new location on first v3.1 launch. ### Artifacts - Master tarball: `ai-lsc-master-2026-07-07.tar.gz` -- New scripts: `scripts/backfill_layer_flags.py` +- New scripts (v3.1): the three `scripts/backfill_*.py` utilities — since retired in the MoE QA pass. - New widgets: `src/ai_lsc/ui/widgets/pipeline_ticker.py`, `src/ai_lsc/ui/widgets/workspace_tab.py` - New docs: `CHANGES.md` (this file), `docs/ADR-002-pipeline-ticker.md`, `docs/ADR-003-workspace-tab.md` - Skipped-items register: `whatremains.txt` diff --git a/QA-PASS-REPORT.md b/QA-PASS-REPORT.md index 5e2ec88..a56b8cf 100644 --- a/QA-PASS-REPORT.md +++ b/QA-PASS-REPORT.md @@ -23,7 +23,7 @@ All instances rewritten as decisive statements. Source files touched: |---|---|---| | `src/ai_lsc/registry/layers/routing.py` | docstring | "Restored layer (v3.1.1b)" → "L5 (Intelligent API Routers & Proxies) holds …"; added source-of-truth note. | | `TODO.md` | 6 | "restored in full from the v3.1.1b routing tarball" → "the canonical 1.3 k-line implementation now lives in-tree". | -| `scripts/apply_10layer_taxonomy.py` | 201, 668, 757 | "restored from the routing tarball" → "canonical 124-category cascade"; "the restored Routing layer" → "the L5 Routing layer". | +| `scripts/apply_10layer_taxonomy.py` (now retired) | 201, 668, 757 | "restored from the routing tarball" → "canonical 124-category cascade"; "the restored Routing layer" → "the L5 Routing layer". | | `CHANGES.md` | 224, 254, 269, 321 | "Restored in full from the v3.1.1b routing tarball" → "canonical 1.3 k-line implementation is re-established in-tree"; section header "Routing restored" → "Routing promoted to a first-class layer"; "(restored)" ladder annotation removed. | | `whatremains.txt` | 9 | "the restored installer cmds" → "the canonical installer cmds". | | `docs/ADR-003-workspace-tab.md` | 67 | "The empty-state placeholder is restored" → "the empty-state placeholder is shown again". | @@ -129,3 +129,61 @@ No code-behavior change; the comment makes the architectural boundary explicit s ## 9. Tarball Packaged as `ai-lsc-moe-qa-pass.tar.gz` in `/home/z/my-project/download/`. Contains the full project tree post-cleanup. + +--- + +## 10. Post-pass cleanup (follow-up) + +After the initial MoE QA pass was packaged, two follow-up changes were applied: + +### 10.1 Retirement of one-shot migration scripts + +The following five scripts were one-shot migrations that had already been +applied to the registry/layer files. Per the project-manager perspective +on the MoE panel ("don't leave dead code sitting around"), they have been +removed from the tree: + +- `apply_taxonomy_migration.py` (root) — v3.2 superseded by `apply_10layer_taxonomy.py`. +- `scripts/apply_10layer_taxonomy.py` — taxonomy re-org, applied. +- `scripts/backfill_layer_flags.py` — H-15 flag backfill, applied. +- `scripts/backfill_tool_licenses.py` — license field backfill, applied. +- `scripts/backfill_default_licenses.py` — license default backfill, applied. + +The empty `scripts/` directory was also removed. All historical references +in `CHANGES.md`, `TODO.md`, and `QA-PASS-REPORT.md` were patched so no +doc points at a missing file. + +### 10.2 Open WebUI launcher fix + +`open-webui serve` dropped the `--data-dir` CLI option in a recent +release; the registry's three open-webui launcher commands all used it +and were failing at startup with: + +``` +No such option '--data-dir'. +``` + +The data directory is now set via the `WEBUI_DATA_DIR` environment +variable, which is the supported mechanism in current open-webui. The +hermes_webui launcher also dropped the invalid `--env OLLAMA_BASE_URL=...` +trailer (open-webui reads `OLLAMA_BASE_URL` from the process environment +directly, not via a CLI flag). Three sites patched: + +- `src/ai_lsc/registry/layers/user_interfaces.py` — `openwebui` launcher. +- `src/ai_lsc/registry/layers/user_interfaces.py` — `hermes_webui` launcher. +- `src/ai_lsc/registry/defaults.py` — `openwebui` launcher. + +Each launcher now reads (e.g. for the canonical openwebui): + +``` +WEBUI_DATA_DIR={workspaces_root}/openwebui open-webui serve --port {port} +``` + +For hermes_webui the env-prefix also carries `OLLAMA_BASE_URL`: + +``` +WEBUI_DATA_DIR={workspaces_root}/hermes-webui OLLAMA_BASE_URL=http://localhost:17051 open-webui serve --port {port} +``` + +The ai-lsc runtime resolves env-prefix shells via `shlex.split()` on the +launcher cmd, so the env-assignment form is parsed correctly. diff --git a/TODO.md b/TODO.md index 711fd3c..25260fa 100755 --- a/TODO.md +++ b/TODO.md @@ -13,16 +13,15 @@ - ✅ Applied the full master code critique: 91 of 93 findings addressed (CRITICAL × 6, HIGH × 24, MEDIUM × 42, LOW × 19). The 2 intentionally skipped items (curl|sh remote installers) are documented in [whatremains.txt](whatremains.txt). - ✅ Added the Pipeline Ticker: scrolling wiring-topology status bar at the top of every workspace tab. Color-coded arrows by interface type, orphans flagged red, click-to-jump to Tools tab. - ✅ Added the Workspace Tab: peek-style orchestration surface with one sub-tab per active tool. Web tools embed via QWebEngineView, CLI tools attach to tmux via `capture-pane` polling. Feels like virt-manager / aqemu for your AI stack. -- ✅ Strengthened the registry validator to enforce the full 8-key flags schema; backfilled 123 layer-file flag blocks via `scripts/backfill_layer_flags.py`. +- ✅ Strengthened the registry validator to enforce the full 8-key flags schema; backfilled 123 layer-file flag blocks (one-shot migration script retired post-apply). - ✅ Reconciled tool count: 124 tools in `defaults.py` + 123 tools across the 13 layer files (merged). ## Open work The 10-layer taxonomy is now in place; classification quality is the next frontier. The 78 layer-file tools absent from the master registry were -classified by hand for v3.2 (table in -`scripts/apply_10layer_taxonomy.py`); review those assignments if any -feel wrong — `layer`/`level` sync from layer files makes corrections a +classified by hand for v3.2 — review those assignments if any feel +wrong: `layer`/`level` sync from layer files makes corrections a one-file edit. If you're reading this file, first of all thanks for trying it out — help by testing the UI, tools, configs, and pipelines. It's a lot. @@ -46,8 +45,8 @@ See [whatremains.txt](whatremains.txt) for the full list. Highlights: - `curl|sh` remote installers (Ollama, Grafana Alloy, Meilisearch) — left in place per user instruction. Apply the critique's download-first pattern when the remote-code-execution policy is revisited. - L-17: `registry/openengineer/parser.py` still has commented-out code blocks — confirm with the OE importer maintainer before deleting. - L-18: registry layer files declare tools without `filesystem` blocks; backfilling `install/config/cache/logs` paths across the layer-file tools is a mechanical but sizable job (v3.2 preserved the 24 defaults.py blocks + n8n/odysseus). -- M-22 / M-40: `chatbot_console.py` HTML bubble builder + nested ternary — could extract `_render_bubble(msg)` helper, deferred as low-impact polish. -- M-10 / M-15 / M-16: `registry/openengineer/importer.py` nested-if flattening + dedup — defer to a focused OE-importer cleanup pass. +- M-22 / M-40: `chatbot_console.py` HTML bubble builder + nested ternary — addressed in the MoE QA pass (`_render_bubble` helper extracted, nested ternary flattened to `"\n\n".join`). +- M-10 / M-15 / M-16: `registry/openengineer/importer.py` nested-if flattening + dedup — addressed in the MoE QA pass (two scan loops merged into single `_scan_dir` helper). ### UX polish still on the wishlist diff --git a/apply_taxonomy_migration.py b/apply_taxonomy_migration.py deleted file mode 100755 index 6dd36da..0000000 --- a/apply_taxonomy_migration.py +++ /dev/null @@ -1,300 +0,0 @@ -#!/usr/bin/env python3 -""" -DCOS.net AI-LSC Codebase Reorganization & 10-Layer Migration Utility -Author: Jeremy Anderson -Domain: dcos.net -Contact: info@dcos.net -License: CC-BY-SA 4.0 International - -This script programmatically upgrades the entire ai_lsc codebase to be -10-layer compliant. It reorganizes layer order, updates the category map, -and ensures that all stack wiring loops dynamically look up layer metadata. -""" - -import os -import re -from pathlib import Path - -# ── 1. Reorganized 10-Layer Taxonomy ─────────────────────────────────────── -NEW_LAYERS = [ - "Host Platform & Infrastructure", - "Development Runtime & Environment", - "GPU Acceleration & Optimization", - "Local Inference Engines", - "Intelligent API Routers & Proxies", - "Multi-Agent Orchestration Runtimes", - "Agentic Software Engineering & Sandboxes", - "Decentralized Knowledge & Vector Stores", - "Data Extraction & Pipeline Harvest", - "Human Interface & System Operations" -] - -# ── 2. Reorganized CATEGORY_MAP ───────────────────────────────────────────── -NEW_CATEGORY_MAP = { - "Agent Daemon": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Agent Daemon"}, - "Agent Shell": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Agent Shell"}, - "Agentless Deployment": {"layer": "Human Interface & System Operations", "level": 10, "role": "Agentless Deployment"}, - "Alloy Telemetry": {"layer": "Human Interface & System Operations", "level": 10, "role": "Alloy Telemetry"}, - "Analytical Database": {"layer": "Host Platform & Infrastructure", "level": 1, "role": "Analytical Database"}, - "Audio Extraction": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Audio Extraction"}, - "Auto Recovery Daemon": {"layer": "Human Interface & System Operations", "level": 10, "role": "Auto Recovery Daemon"}, - "CI/CD Orchestration": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "CI/CD Orchestration"}, - "CLI Pair Programmer": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "CLI Pair Programmer"}, - "Cache": {"layer": "Host Platform & Infrastructure", "level": 1, "role": "Cache"}, - "Cloud Dev Kit": {"layer": "Human Interface & System Operations", "level": 10, "role": "Cloud Dev Kit"}, - "Cluster Homepage": {"layer": "Human Interface & System Operations", "level": 10, "role": "Cluster Homepage"}, - "Code Refactoring": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Code Refactoring"}, - "Codex Developer": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Codex Developer"}, - "Collaborative Chat": {"layer": "Human Interface & System Operations", "level": 10, "role": "Collaborative Chat"}, - "Context Integrity Core": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Context Integrity Core"}, - "Coordination Protocol": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Coordination Protocol"}, - "Curation Pipeline": {"layer": "Intelligent API Routers & Proxies", "level": 5, "role": "Curation Pipeline"}, - "Data Ingest": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Data Ingest"}, - "Database": {"layer": "Host Platform & Infrastructure", "level": 1, "role": "Database"}, - "Datacenter Inference": {"layer": "Local Inference Engines", "level": 4, "role": "Datacenter Inference"}, - "Desktop Agent Core": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Desktop Agent Core"}, - "Desktop GUI": {"layer": "Human Interface & System Operations", "level": 10, "role": "Desktop GUI"}, - "Distributed Search": {"layer": "Decentralized Knowledge & Vector Stores", "level": 8, "role": "Distributed Search"}, - "Document Archiver": {"layer": "Human Interface & System Operations", "level": 10, "role": "Document Archiver"}, - "Document Comprehension": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Document Comprehension"}, - "Document Parser": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Document Parser"}, - "ETL Transformation": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "ETL Transformation"}, - "Ecosystem Launcher": {"layer": "Human Interface & System Operations", "level": 10, "role": "Ecosystem Launcher"}, - "Enterprise Assistant": {"layer": "Human Interface & System Operations", "level": 10, "role": "Enterprise Assistant"}, - "Extensible Interface": {"layer": "Human Interface & System Operations", "level": 10, "role": "Extensible Interface"}, - "File Discovery": {"layer": "Development Runtime & Environment", "level": 2, "role": "File Discovery"}, - "GGUF Runtime": {"layer": "Local Inference Engines", "level": 4, "role": "GGUF Runtime"}, - "GPU Computing": {"layer": "GPU Acceleration & Optimization", "level": 3, "role": "GPU Computing"}, - "Gap Analyzer": {"layer": "Human Interface & System Operations", "level": 10, "role": "Gap Analyzer"}, - "Graph Database": {"layer": "Decentralized Knowledge & Vector Stores", "level": 8, "role": "Graph Database"}, - "Hardware Acceleration": {"layer": "GPU Acceleration & Optimization", "level": 3, "role": "Hardware Acceleration"}, - "High-Performance Vector": {"layer": "Decentralized Knowledge & Vector Stores", "level": 8, "role": "High-Performance Vector"}, - "IaC DRY Wrapper": {"layer": "Human Interface & System Operations", "level": 10, "role": "IaC DRY Wrapper"}, - "Image WebUI": {"layer": "Human Interface & System Operations", "level": 10, "role": "Image WebUI"}, - "Infrastructure Provision": {"layer": "Human Interface & System Operations", "level": 10, "role": "Infrastructure Provision"}, - "Infrastructure as Code": {"layer": "Development Runtime & Environment", "level": 2, "role": "Infrastructure as Code"}, - "Integration Library": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Integration Library"}, - "Isolation": {"layer": "Host Platform & Infrastructure", "level": 1, "role": "Isolation"}, - "LLM Observability": {"layer": "Human Interface & System Operations", "level": 10, "role": "LLM Observability"}, - "Layer-wise Inference": {"layer": "Local Inference Engines", "level": 4, "role": "Layer-wise Inference"}, - "Learning Cards": {"layer": "Human Interface & System Operations", "level": 10, "role": "Learning Cards"}, - "Lexical Search": {"layer": "Decentralized Knowledge & Vector Stores", "level": 8, "role": "Lexical Search"}, - "Library Manager": {"layer": "Human Interface & System Operations", "level": 10, "role": "Library Manager"}, - "Load Balancer": {"layer": "Intelligent API Routers & Proxies", "level": 5, "role": "Load Balancer"}, - "Local Central Shell": {"layer": "Human Interface & System Operations", "level": 10, "role": "Local Central Shell"}, - "Local Declarative DSL": {"layer": "Human Interface & System Operations", "level": 10, "role": "Local Declarative DSL"}, - "Local LLM Runner": {"layer": "Local Inference Engines", "level": 4, "role": "Local LLM Runner"}, - "Markdown Converter": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Markdown Converter"}, - "Metal Provisioner": {"layer": "Human Interface & System Operations", "level": 10, "role": "Metal Provisioner"}, - "Metric Scraper": {"layer": "Human Interface & System Operations", "level": 10, "role": "Metric Scraper"}, - "Mixed Precision": {"layer": "GPU Acceleration & Optimization", "level": 3, "role": "Mixed Precision"}, - "Model Curation": {"layer": "Local Inference Engines", "level": 4, "role": "Model Curation"}, - "Model Manager GUI": {"layer": "Human Interface & System Operations", "level": 10, "role": "Model Manager GUI"}, - "Model Optimization": {"layer": "GPU Acceleration & Optimization", "level": 3, "role": "Model Optimization"}, - "Model Surgery": {"layer": "Local Inference Engines", "level": 4, "role": "Model Surgery"}, - "Multi-Agent Framework": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Multi-Agent Framework"}, - "Native Inference": {"layer": "Local Inference Engines", "level": 4, "role": "Native Inference"}, - "Offline Journal": {"layer": "Human Interface & System Operations", "level": 10, "role": "Offline Journal"}, - "Open-Source IaC": {"layer": "Human Interface & System Operations", "level": 10, "role": "Open-Source IaC"}, - "Outliner Graph": {"layer": "Human Interface & System Operations", "level": 10, "role": "Outliner Graph"}, - "Output Evaluator": {"layer": "Human Interface & System Operations", "level": 10, "role": "Output Evaluator"}, - "PDF Processing": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "PDF Processing"}, - "Parsing Engine": {"layer": "Development Runtime & Environment", "level": 2, "role": "Parsing Engine"}, - "Performance Engine": {"layer": "Local Inference Engines", "level": 4, "role": "Performance Engine"}, - "Platform Workspace": {"layer": "Human Interface & System Operations", "level": 10, "role": "Platform Workspace"}, - "Production Multi-Agent": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Production Multi-Agent"}, - "Programmable IaC": {"layer": "Human Interface & System Operations", "level": 10, "role": "Programmable IaC"}, - "Prompt Tester": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Prompt Tester"}, - "RAG Flow Canvas": {"layer": "Human Interface & System Operations", "level": 10, "role": "RAG Flow Canvas"}, - "Reference Manager": {"layer": "Human Interface & System Operations", "level": 10, "role": "Reference Manager"}, - "Requirement Builder": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Requirement Builder"}, - "Role-Playing Agent": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Role-Playing Agent"}, - "Runtime Environment": {"layer": "Development Runtime & Environment", "level": 2, "role": "Runtime Environment"}, - "Runtime Packaging": {"layer": "Host Platform & Infrastructure", "level": 1, "role": "Runtime Packaging"}, - "Serverless Database": {"layer": "Decentralized Knowledge & Vector Stores", "level": 8, "role": "Serverless Database"}, - "Single-File Runtime": {"layer": "Local Inference Engines", "level": 4, "role": "Single-File Runtime"}, - "Skill Auditor": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Skill Auditor"}, - "Sovereign Graph Editor": {"layer": "Human Interface & System Operations", "level": 10, "role": "Sovereign Graph Editor"}, - "Sovereign Software Agent": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Sovereign Software Agent"}, - "Speech Transcriber": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Speech Transcriber"}, - "Sprint Planner": {"layer": "Human Interface & System Operations", "level": 10, "role": "Sprint Planner"}, - "Sprints Manager": {"layer": "Human Interface & System Operations", "level": 10, "role": "Sprints Manager"}, - "Stateless Orchestration": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Stateless Orchestration"}, - "Structured Synthesis": {"layer": "Decentralized Knowledge & Vector Stores", "level": 8, "role": "Structured Synthesis"}, - "Studio Canvas": {"layer": "Human Interface & System Operations", "level": 10, "role": "Studio Canvas"}, - "Synchronization Layer": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Synchronization Layer"}, - "System Auditor": {"layer": "Human Interface & System Operations", "level": 10, "role": "System Auditor"}, - "System Console": {"layer": "Human Interface & System Operations", "level": 10, "role": "System Console"}, - "Telemetry Monitor": {"layer": "Human Interface & System Operations", "level": 10, "role": "Telemetry Monitor"}, - "Telemetry Visualizer": {"layer": "Human Interface & System Operations", "level": 10, "role": "Telemetry Visualizer"}, - "Terminal Agent": {"layer": "Multi-Agent Orchestration Runtimes", "level": 6, "role": "Terminal Agent"}, - "Terminal Coding Agent": {"layer": "Agentic Software Engineering & Sandboxes", "level": 7, "role": "Terminal Coding Agent"}, - "Text Search": {"layer": "Development Runtime & Environment", "level": 2, "role": "Text Search"}, - "Unified API Gateway": {"layer": "Intelligent API Routers & Proxies", "level": 5, "role": "Unified API Gateway"}, - "Vector Engine": {"layer": "Decentralized Knowledge & Vector Stores", "level": 8, "role": "Vector Engine"}, - "Vision describer": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Vision describer"}, - "Visual Agent Canvas": {"layer": "Human Interface & System Operations", "level": 10, "role": "Visual Agent Canvas"}, - "Voice Synthesis": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Voice Synthesis"}, - "Web Scraper": {"layer": "Data Extraction & Pipeline Harvest", "level": 9, "role": "Web Scraper"}, -} - -def migrate_constants(path: Path): - if not path.exists(): - print(f"[-] constants.py not found at {path}") - return - content = path.read_text(encoding="utf-8") - - # Locate NAV_LAYER_ORDER array - nav_pattern = re.compile(r'NAV_LAYER_ORDER:\s*list\[str\]\s*=\s*\[[^\]]+\]', re.DOTALL) - new_nav_str = "NAV_LAYER_ORDER: list[str] = [\n" + "".join(f' "{l}",\n' for l in NEW_LAYERS) + "]" - - if nav_pattern.search(content): - content = nav_pattern.sub(new_nav_str, content) - path.write_text(content, encoding="utf-8") - print(f"[+] Migrated constants.py layers!") - else: - print(f"[-] NAV_LAYER_ORDER block not matched in constants.py") - -def migrate_db_manager(path: Path): - if not path.exists(): - print(f"[-] db_manager.py not found at {path}") - return - content = path.read_text(encoding="utf-8") - - # Locate CATEGORY_MAP dict block - map_pattern = re.compile(r'CATEGORY_MAP:\s*dict\[str,\s*dict\[str,\s*object\]\]\s*=\s*\{[^}]+(\}[^}]*)*\}', re.DOTALL) - - # Format CATEGORY_MAP dict - new_map_str = "CATEGORY_MAP: dict[str, dict[str, object]] = {\n" - for cat in sorted(NEW_CATEGORY_MAP.keys()): - val = NEW_CATEGORY_MAP[cat] - new_map_str += f' "{cat}": ' + "{" + f'"layer": "{val["layer"]}", "level": {val["level"]}, "role": "{val["role"]}"' + "},\n" - new_map_str += "}" - - if map_pattern.search(content): - content = map_pattern.sub(new_map_str, content) - path.write_text(content, encoding="utf-8") - print(f"[+] Migrated db_manager.py CATEGORY_MAP!") - else: - print(f"[-] CATEGORY_MAP block not matched in db_manager.py") - -def migrate_connections(path: Path, defaults_path: Path): - if not path.exists(): - print(f"[-] connections.py not found at {path}") - return - if not defaults_path.exists(): - print(f"[-] defaults.py not found at {defaults_path} - cannot resolve dynamic tool layers") - return - - # Dynamically read tool_id -> layer from defaults.py - sys.path.insert(0, str(defaults_path.parent)) - import defaults - tool_to_layer = {} - for tid, tool in defaults.DEFAULT_REGISTRY.items(): - tool_to_layer[tid] = tool["layer"] - - content = path.read_text(encoding="utf-8") - - # Add DEFAULT_REGISTRY import to connections.py if missing - import_line = "from ai_lsc.registry.defaults import DEFAULT_REGISTRY" - if "import DEFAULT_REGISTRY" not in content and import_line not in content: - # Insert after absolute imports - content = "from ai_lsc.registry.defaults import DEFAULT_REGISTRY\n" + content - print("[+] Added DEFAULT_REGISTRY import to connections.py") - - # Reorganize the StackWiring layer arguments in connections.py - blocks = content.split("_reg(StackWiring(") - new_blocks = [blocks[0]] - updated_count = 0 - - for block in blocks[1:]: - tid_match = re.search(r"""tool_id\s*=\s*["']?([a-zA-Z0-9_-]+)["']?""", block) - if tid_match: - tid = tid_match.group(1) - if tid in tool_to_layer: - new_layer = tool_to_layer[tid] - # Replace layer="..." or layer='...' - block, count = re.subn(r"""layer\s*=\s*["|'][^"'\s]+["|']""", f'layer="{new_layer}"', block) - if count: - updated_count += 1 - new_blocks.append(block) - - content = "_reg(StackWiring(".join(new_blocks) - - # Convert hardcoded loops to look up the layer from the registry dynamically - # Example: layer="DevOps" inside loops -> layer=DEFAULT_REGISTRY[_tid]["layer"] - content, loops_updated = re.subn( - r"""layer\s*=\s*["'](?:DevOps|User Interfaces|Knowledge Management|Orchestrators|Host Platform|Development Environment|GPU Runtimes|Engines|Routing|Security|Observability)["']""", - 'layer=DEFAULT_REGISTRY[_tid]["layer"]', - content - ) - if loops_updated: - print(f"[+] Converted {loops_updated} loop-based StackWiring layers to dynamic lookups!") - - path.write_text(content, encoding="utf-8") - print(f"[+] Migrated connections.py! Rebuilt {updated_count} static tool-layer definitions.") - -def migrate_layers(layers_dir: Path, defaults_path: Path): - if not layers_dir.is_dir(): - print(f"[-] Layers directory not found at {layers_dir}") - return - if not defaults_path.exists(): - print(f"[-] defaults.py not found - cannot update layer files") - return - - import defaults - tool_to_layer = {} - tool_to_level = {} - for tid, tool in defaults.DEFAULT_REGISTRY.items(): - tool_to_layer[tid] = tool["layer"] - tool_to_level[tid] = tool["level"] - - updated_files = 0 - for path in layers_dir.glob("*.py"): - if path.name == "__init__.py": - continue - original = path.read_text(encoding="utf-8") - content = original - - # Replace level and layer for each tool inside the file - for tid in tool_to_layer: - # Look for block: 'tool_id': { or "tool_id": { - # we do a specific match and replace within that dict - tool_pattern = re.compile(rf"""["']{tid}["']\s*:\s*\{{[^}}]+}}""", re.DOTALL) - match = tool_pattern.search(content) - if match: - block = match.group(0) - # Replace level - block = re.sub(r'"level":\s*\d+', f'"level": {tool_to_level[tid]}', block) - # Replace layer - block = re.sub(r""""layer":\s*["|'][^"'\s]+["|']""", f'"layer": "{tool_to_layer[tid]}"', block) - content = content.replace(match.group(0), block) - - if content != original: - path.write_text(content, encoding="utf-8") - print(f"[+] Realigned tool layers & levels inside modular layer file: {path.name}") - updated_files += 1 - - print(f"[+] Layer files update complete. Modified {updated_files} layer modules.") - -def main(): - import sys - root = Path(__file__).resolve().parent - src_dir = root / "src" - - # Setup paths relative to root or src - constants_path = src_dir / "ai_lsc" / "constants.py" - db_manager_path = src_dir / "ai_lsc" / "ui" / "pages" / "db_manager.py" - connections_path = src_dir / "ai_lsc" / "stack" / "connections.py" - defaults_path = src_dir / "ai_lsc" / "registry" / "defaults.py" - layers_dir = src_dir / "ai_lsc" / "registry" / "layers" - - print("[*] Starting taxonomy migration sweeps...") - migrate_constants(constants_path) - migrate_db_manager(db_manager_path) - migrate_connections(connections_path, defaults_path) - migrate_layers(layers_dir, defaults_path) - print("[+] Entire codebase has been successfully upgraded to the 10-Layer Architecture!") - -if __name__ == "__main__": - main() diff --git a/src/ai_lsc/registry/defaults.py b/src/ai_lsc/registry/defaults.py index 70dd1cb..2639263 100755 --- a/src/ai_lsc/registry/defaults.py +++ b/src/ai_lsc/registry/defaults.py @@ -3132,7 +3132,7 @@ DEFAULT_REGISTRY: dict = { }, "launcher": { "type": 'tmux', - "cmd": 'open-webui serve --port {port} --data-dir {workspaces_root}/openwebui', + "cmd": 'WEBUI_DATA_DIR={workspaces_root}/openwebui open-webui serve --port {port}', "default_port": 8080, }, "deps": [ diff --git a/src/ai_lsc/registry/layers/user_interfaces.py b/src/ai_lsc/registry/layers/user_interfaces.py index c7f6f58..86f5769 100755 --- a/src/ai_lsc/registry/layers/user_interfaces.py +++ b/src/ai_lsc/registry/layers/user_interfaces.py @@ -24,7 +24,7 @@ TOOLS: dict[str, dict] = { }, "launcher": { "type": "tmux", - "cmd": "open-webui serve --port {port} --data-dir {workspaces_root}/openwebui", + "cmd": "WEBUI_DATA_DIR={workspaces_root}/openwebui open-webui serve --port {port}", "default_port": 8080 }, "deps": [ @@ -540,7 +540,7 @@ TOOLS: dict[str, dict] = { }, "launcher": { "type": "tmux", - "cmd": "open-webui serve --port {port} --data-dir {workspaces_root}/hermes-webui --env OLLAMA_BASE_URL=http://localhost:17051", + "cmd": "WEBUI_DATA_DIR={workspaces_root}/hermes-webui OLLAMA_BASE_URL=http://localhost:17051 open-webui serve --port {port}", "default_port": 8081 }, "deps": [ diff --git a/src/ai_lsc/stack/connections.py b/src/ai_lsc/stack/connections.py index d68a80a..183e8ce 100755 --- a/src/ai_lsc/stack/connections.py +++ b/src/ai_lsc/stack/connections.py @@ -3398,13 +3398,13 @@ _reg(StackWiring( decision="Run a second Open-WebUI instance dedicated " "to the Hermes stack.", observation="Open-WebUI supports multiple instances " - "with separate data-dir flags. The general " - "openwebui tool already occupies :3000 with " - "Ollama direct as backend.", + "with separate WEBUI_DATA_DIR env vars. The " + "general openwebui tool already occupies :3000 " + "with Ollama direct as backend.", alternatives="Use a single Open-WebUI instance with " "model name prefixes (hermes/*).", constraints="Must run on a different port (8081) and " - "data-dir than the general openwebui.", + "WEBUI_DATA_DIR than the general openwebui.", reasoning="Dedicated Hermes UI keeps the Hermes agent " "runtime as the single backend, so every " "Hermes WebUI conversation flows through "