327 lines
12 KiB
Python
Executable File
327 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Functionality matrix — every destination, live.
|
|
|
|
One PASS/FAIL/SKIP row per target or stage against real services:
|
|
|
|
destinations : obsidian (notes only) + ten vector stores
|
|
per target : ingest → re-ingest idempotency → retrieval assertions
|
|
graphs : LightRAG and Graphify engines (one document each)
|
|
archives : MinIO object stage + filesystem bz2 stage
|
|
ask : Vanna natural-language SQL (pgvector + mariadb)
|
|
|
|
Exit code 0 iff no FAIL. Services read the standard env (PG*,
|
|
MARIADB_*, MINIO_*); missing services degrade to SKIP, never crash.
|
|
|
|
Run: .venv/bin/python scripts/func_test.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
LAUNCHER = ROOT / "thicket.py"
|
|
RESULTS: list[tuple[str, str, str]] = [] # (section, name, outcome)
|
|
|
|
# Local service credentials for the standing test stack.
|
|
os.environ.setdefault("PGHOST", "localhost")
|
|
os.environ.setdefault("PGUSER", "thicket")
|
|
os.environ.setdefault("PGPASSWORD", "thicket")
|
|
os.environ.setdefault("PGDATABASE", "thicket")
|
|
os.environ.setdefault("MARIADB_HOST", "127.0.0.1")
|
|
os.environ.setdefault("MARIADB_USER", "root")
|
|
os.environ.setdefault("MARIADB_DATABASE", "thicket")
|
|
os.environ.setdefault("MINIO_ENDPOINT", "127.0.0.1:9000")
|
|
os.environ.setdefault("MINIO_ACCESS_KEY", "thicket")
|
|
os.environ.setdefault("MINIO_SECRET_KEY", "thicket-secret")
|
|
|
|
EMBED_MODEL = "jinaai/jina-embeddings-v2-base-code"
|
|
COLLECTION = "thicket_func"
|
|
|
|
CORPUS = {
|
|
"tls-rotation.sh": (
|
|
"#!/usr/bin/env bash\n"
|
|
"set -euo pipefail\n"
|
|
"# rotate tls certificates weekly and reload the edge\n"
|
|
"certbot renew --quiet --deploy-hook \"systemctl reload nginx\"\n"
|
|
"find /etc/letsencrypt/archive -mtime +90 -delete\n"
|
|
),
|
|
"timeout-policy.md": (
|
|
"# Edge Timeout Policy\n\n"
|
|
"Production proxies apply strict connection timeouts.\n\n"
|
|
"## nginx block\n\n"
|
|
"```nginx\n"
|
|
"proxy_connect_timeout 300s;\n"
|
|
"proxy_read_timeout 300s;\n"
|
|
"client_max_body_size 25m;\n"
|
|
"```\n\n"
|
|
"The 300 second ceiling bounds slow-loris exposure.\n"
|
|
),
|
|
"pasta.txt": (
|
|
"Kitchen notes: the asparagus pasta with basil pesto needs the\n"
|
|
"noodles pulled a minute early — residual heat finishes them.\n"
|
|
),
|
|
}
|
|
|
|
QUERIES = [ # (query, expected document stem)
|
|
("rotate tls certificates weekly", "tls-rotation"),
|
|
("connection timeout 300 nginx", "timeout-policy"),
|
|
("asparagus pasta recipe", "pasta"),
|
|
]
|
|
|
|
|
|
def record(section: str, name: str, outcome: str, detail: str = "") -> None:
|
|
tag = outcome if not detail else f"{outcome} — {detail}"
|
|
RESULTS.append((section, name, tag))
|
|
print(f" [{outcome:4s}] {name:28s} {detail}")
|
|
|
|
|
|
def run_cli(args: list[str], timeout: int = 600) -> tuple[int, str]:
|
|
proc = subprocess.run(
|
|
[sys.executable, str(LAUNCHER), *args],
|
|
capture_output=True, text=True, timeout=timeout, cwd=ROOT,
|
|
)
|
|
return proc.returncode, proc.stdout + proc.stderr
|
|
|
|
|
|
def make_corpus(base: Path) -> Path:
|
|
docs = base / "cold"
|
|
docs.mkdir(parents=True)
|
|
for name, body in CORPUS.items():
|
|
(docs / name).write_text(body, encoding="utf-8")
|
|
return docs
|
|
|
|
|
|
def fresh_vault(base: Path, name: str) -> Path:
|
|
vault = base / f"vault-{name}"
|
|
vault.mkdir(parents=True, exist_ok=True)
|
|
return vault
|
|
|
|
|
|
def search_once(target: str, vault: Path, query: str, port: int = 6333):
|
|
from thicket.vector_stores import create_store
|
|
|
|
engine = search_once._engine
|
|
store = create_store(target, collection=COLLECTION, dim=engine.dim,
|
|
host="localhost", port=port,
|
|
data_dir=vault / ".thicket" / target)
|
|
store.set_embedder(engine)
|
|
try:
|
|
store.ensure_collection()
|
|
return store.search(engine.embed_query(query), limit=1)
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
def drop_stale() -> None:
|
|
"""The matrix owns its collection name; any earlier state at that
|
|
name (including other dimension eras) goes before the run."""
|
|
try:
|
|
from qdrant_client import QdrantClient
|
|
c = QdrantClient(url="http://localhost:6333",
|
|
check_compatibility=False)
|
|
if c.collection_exists(COLLECTION):
|
|
c.delete_collection(COLLECTION)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import psycopg
|
|
with psycopg.connect(connect_timeout=3) as conn:
|
|
conn.execute(f'DROP TABLE IF EXISTS "{COLLECTION}"')
|
|
conn.commit()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import pymysql
|
|
conn = pymysql.connect(host="127.0.0.1", user="root",
|
|
database="thicket", connect_timeout=3)
|
|
with conn.cursor() as cur:
|
|
cur.execute(f"DROP TABLE IF EXISTS `{COLLECTION}`")
|
|
conn.commit(); conn.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import weaviate
|
|
client = weaviate.connect_to_local(
|
|
host="127.0.0.1", port=8080, grpc_port=50051)
|
|
mapped = "Thicket_func"
|
|
if client.collections.exists(mapped):
|
|
client.collections.delete(mapped)
|
|
client.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def phase_destinations(base: Path) -> None:
|
|
from thicket.embedder import EmbeddingEngine
|
|
from thicket.vector_stores import TARGETS
|
|
|
|
drop_stale()
|
|
engine = EmbeddingEngine(EMBED_MODEL)
|
|
engine.load()
|
|
search_once._engine = engine
|
|
|
|
print("\n== destinations ==")
|
|
# obsidian: notes only
|
|
docs = make_corpus(base / "obsidian")
|
|
vault = fresh_vault(base, "obsidian")
|
|
rc, out = run_cli(["--ingest", str(docs), "--vault", str(vault),
|
|
"--target", "obsidian", "--no-qdrant"])
|
|
notes = sorted(p.name for p in (vault / "Ingested_Brain").glob("*.md"))
|
|
ok = rc == 0 and len(notes) == 3 and "INDEX" not in out
|
|
record("destinations", "obsidian",
|
|
"PASS" if ok else "FAIL", f"notes={notes}" if ok else out[-160:])
|
|
|
|
parity: dict[str, float] = {}
|
|
for target in TARGETS:
|
|
docs = make_corpus(base / target)
|
|
vault = fresh_vault(base, target)
|
|
port = 8080 if target == "weaviate" else 6333
|
|
rc, out = run_cli(["--ingest", str(docs), "--vault", str(vault),
|
|
"--target", target, "--port", str(port),
|
|
"--collection", COLLECTION])
|
|
if rc != 0 or "0 failed" not in out:
|
|
record("destinations", target, "FAIL", out.strip()[-140:])
|
|
continue
|
|
try:
|
|
first = search_once(target, vault, QUERIES[0][0], port)
|
|
rc2, _ = run_cli(["--ingest", str(docs), "--vault", str(vault),
|
|
"--target", target, "--port", str(port),
|
|
"--collection", COLLECTION])
|
|
second = search_once(target, vault, QUERIES[0][0], port)
|
|
except Exception as e: # noqa: BLE001
|
|
record("destinations", target, "FAIL", f"search: {e}")
|
|
continue
|
|
hit_ok = all(
|
|
search_once(target, vault, q, port)[0]["payload"]
|
|
["document_title"].lower().replace(" ", "-").startswith(exp.split("-")[0])
|
|
for q, exp in QUERIES[:1]
|
|
)
|
|
idem = (rc2 == 0 and abs(first[0]["score"] - second[0]["score"]) < 1e-3
|
|
and second[0]["payload"]["document_title"]
|
|
== first[0]["payload"]["document_title"])
|
|
parity[target] = first[0]["score"]
|
|
record("destinations", target,
|
|
"PASS" if (hit_ok and idem) else "FAIL",
|
|
f"top={first[0]['payload']['document_title'][:22]!r} "
|
|
f"score={first[0]['score']:.3f} idempotent={idem}")
|
|
|
|
if len(parity) >= 2:
|
|
scores = sorted(parity.values())
|
|
spread = scores[-1] - scores[0]
|
|
record("destinations", "score parity (all)",
|
|
"PASS" if spread <= 0.02 else "WARN", f"spread={spread:.4f}")
|
|
|
|
|
|
def phase_graphs(base: Path) -> None:
|
|
print("\n== graph engines ==")
|
|
for engine, marker in (("lightrag", ".lightrag"), ("graphify", ".graphify")):
|
|
docs = make_corpus(base / f"g-{engine}")
|
|
docs_extra = docs / "tls-rotation.sh"
|
|
docs_extra.write_text(CORPUS["tls-rotation.sh"], encoding="utf-8")
|
|
vault = fresh_vault(base, f"g-{engine}")
|
|
rc, out = run_cli(["--ingest", str(docs), "--vault", str(vault),
|
|
"--target", "obsidian", "--no-qdrant",
|
|
"--lightrag", "--graph-engine", engine,
|
|
"--ollama-llm", "llama3.1:latest"],
|
|
timeout=900)
|
|
if rc != 0:
|
|
record("graphs", engine, "FAIL", out.strip()[-160:])
|
|
continue
|
|
if engine == "graphify":
|
|
ok = (vault / ".graphify" / "graphify-out" / "graph.json").exists()
|
|
else:
|
|
graph_dir = vault / ".lightrag"
|
|
ok = graph_dir.exists() and any(graph_dir.iterdir())
|
|
record("graphs", engine, "PASS" if ok else "FAIL",
|
|
f"artifacts={'yes' if ok else 'missing'}")
|
|
|
|
|
|
def phase_archives(base: Path) -> None:
|
|
import bz2
|
|
|
|
print("\n== archive stages ==")
|
|
# filesystem bz2
|
|
docs = make_corpus(base / "fsarch")
|
|
vault = fresh_vault(base, "fsarch")
|
|
archive = base / "ingested-archive"
|
|
rc, out = run_cli(["--ingest", str(docs), "--vault", str(vault),
|
|
"--target", "obsidian", "--no-qdrant", "--archive",
|
|
"--archive-dir", str(archive)])
|
|
packed = sorted(archive.glob("*.bz2")) if archive.exists() else []
|
|
roundtrip = bool(packed) and all(
|
|
bz2.decompress(p.read_bytes()) for p in packed)
|
|
empty_in = not any(docs.iterdir())
|
|
record("archives", "filesystem bz2",
|
|
"PASS" if (rc == 0 and roundtrip and empty_in) else "FAIL",
|
|
f"{len(packed)} objects, roundtrip={roundtrip}")
|
|
|
|
# MinIO object stage
|
|
try:
|
|
from minio import Minio
|
|
client = Minio("127.0.0.1:9000", access_key="thicket",
|
|
secret_key="thicket-secret", secure=False)
|
|
if not client.bucket_exists("thicket-corpus"):
|
|
client.make_bucket("thicket-corpus")
|
|
except Exception as e: # noqa: BLE001
|
|
record("archives", "minio objects", "SKIP", f"service: {e}")
|
|
return
|
|
docs = make_corpus(base / "minioarch")
|
|
vault = fresh_vault(base, "minioarch")
|
|
rc, out = run_cli(["--ingest", str(docs), "--vault", str(vault),
|
|
"--target", "obsidian", "--no-qdrant", "--minio"])
|
|
objects = {o.object_name for o in
|
|
client.list_objects("thicket-corpus", recursive=True)}
|
|
uris = "source_uri" in (vault / "Ingested_Brain" / "pasta.md").read_text()
|
|
expected = set(CORPUS)
|
|
ok = rc == 0 and uris and expected & objects == expected
|
|
record("archives", "minio objects",
|
|
"PASS" if ok else "FAIL",
|
|
f"{len(expected & objects)}/{len(expected)} objects, uri_in_note={uris}")
|
|
|
|
|
|
def phase_ask(base: Path) -> None:
|
|
print("\n== ask (Vanna + Ollama) ==")
|
|
from thicket.ask_vanna import SQL_TARGETS, ask
|
|
|
|
for target in SQL_TARGETS:
|
|
try:
|
|
answer = ask("How many rows are in the corpus table?",
|
|
target=target, llm_model="llama3.1:latest")
|
|
ok = bool(answer.strip()) and "ERROR" not in answer
|
|
record("ask", target, "PASS" if ok else "FAIL",
|
|
answer.strip().replace("\n", " ")[:60])
|
|
except Exception as e: # noqa: BLE001
|
|
record("ask", target, "FAIL", str(e)[:100])
|
|
|
|
|
|
def main() -> int:
|
|
started = time.time()
|
|
print(f"thicket functionality matrix — {time.strftime('%Y-%m-%d %H:%M')}")
|
|
base = Path(tempfile.mkdtemp(prefix="thicket-func-"))
|
|
try:
|
|
phase_destinations(base)
|
|
phase_graphs(base)
|
|
phase_archives(base)
|
|
phase_ask(base)
|
|
finally:
|
|
shutil.rmtree(base, ignore_errors=True)
|
|
|
|
fails = [r for r in RESULTS if r[2].startswith("FAIL")]
|
|
print(f"\n{'=' * 60}")
|
|
print(f"{len(RESULTS)} checks, {len(fails)} failed "
|
|
f"({time.time() - started:.0f}s)")
|
|
return 1 if fails else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|