"""MinIO object archive — durable cold storage for source documents. The source files themselves (PDFs, EPUBs, notes) are uploaded to an S3-compatible MinIO bucket as part of ingest; the vault note records the resulting ``s3://bucket/key`` URI in its frontmatter. The vault and vector index stay lean; the originals live in object storage. Open-source MinIO has no vector API (vector search is an AIStor feature; the community edition is S3 object storage) — so MinIO is an archive stage here, never a vector target. Connection from Unix-standard env: MINIO_ENDPOINT (host:port), MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE (default false), MINIO_BUCKET (default thicket-corpus). """ from __future__ import annotations import os from collections.abc import Callable from pathlib import Path DEFAULT_BUCKET = "thicket-corpus" class ArchiveUnavailable(Exception): """Raised when the archive stage cannot be constructed or run.""" class MinioArchiver: """Uploads source documents to a MinIO bucket. Same key = same document: re-ingesting overwrites in place (idempotent).""" def __init__(self, log: Callable[[str], None] = lambda _msg: None): try: from minio import Minio except ImportError as e: raise ArchiveUnavailable( "minio package not installed — run: pip install 'thicket[minio]'" ) from e endpoint = os.environ.get("MINIO_ENDPOINT", "localhost:9000") self.bucket = os.environ.get("MINIO_BUCKET", DEFAULT_BUCKET) secure = os.environ.get("MINIO_SECURE", "").lower() in ("1", "true", "yes") self._client = Minio( endpoint, access_key=os.environ.get("MINIO_ACCESS_KEY", ""), secret_key=os.environ.get("MINIO_SECRET_KEY", ""), secure=secure, ) self._log = log try: if not self._client.bucket_exists(self.bucket): self._log(f"Creating MinIO bucket '{self.bucket}'...") self._client.make_bucket(self.bucket) except Exception as e: raise ArchiveUnavailable( f"cannot reach MinIO at {endpoint}: {e} — set " f"MINIO_ENDPOINT / MINIO_ACCESS_KEY / MINIO_SECRET_KEY" ) from e def object_key(self, source_path: Path, input_dir: Path) -> str: """Bucket key mirrors the input tree's relative path.""" return source_path.relative_to(input_dir).as_posix() def archive(self, source_path: Path, input_dir: Path) -> str: """Upload one source document; returns its s3:// URI.""" key = self.object_key(source_path, input_dir) try: self._client.fput_object( self.bucket, key, str(source_path), ) except Exception as e: raise ArchiveUnavailable( f"upload of '{key}' failed: {e}" ) from e return f"s3://{self.bucket}/{key}"