78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
"""MinIO object archive — durable cold storage for source documents.
|
|
|
|
The source files themselves (PDFs, EPUBs, notes) are uploaded to an
|
|
S3-compatible MinIO bucket as part of ingest; the vault note records
|
|
the resulting ``s3://bucket/key`` URI in its frontmatter. The vault
|
|
and vector index stay lean; the originals live in object storage.
|
|
|
|
Open-source MinIO has no vector API (vector search is an AIStor
|
|
feature; the community edition is S3 object storage) — so MinIO is an
|
|
archive stage here, never a vector target.
|
|
|
|
Connection from Unix-standard env: MINIO_ENDPOINT (host:port),
|
|
MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE (default false),
|
|
MINIO_BUCKET (default thicket-corpus).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
|
|
DEFAULT_BUCKET = "thicket-corpus"
|
|
|
|
|
|
class ArchiveUnavailable(Exception):
|
|
"""Raised when the archive stage cannot be constructed or run."""
|
|
|
|
|
|
class MinioArchiver:
|
|
"""Uploads source documents to a MinIO bucket. Same key = same
|
|
document: re-ingesting overwrites in place (idempotent)."""
|
|
|
|
def __init__(self, log: Callable[[str], None] = lambda _msg: None):
|
|
try:
|
|
from minio import Minio
|
|
except ImportError as e:
|
|
raise ArchiveUnavailable(
|
|
"minio package not installed — run: pip install 'thicket[minio]'"
|
|
) from e
|
|
|
|
endpoint = os.environ.get("MINIO_ENDPOINT", "localhost:9000")
|
|
self.bucket = os.environ.get("MINIO_BUCKET", DEFAULT_BUCKET)
|
|
secure = os.environ.get("MINIO_SECURE", "").lower() in ("1", "true", "yes")
|
|
self._client = Minio(
|
|
endpoint,
|
|
access_key=os.environ.get("MINIO_ACCESS_KEY", ""),
|
|
secret_key=os.environ.get("MINIO_SECRET_KEY", ""),
|
|
secure=secure,
|
|
)
|
|
self._log = log
|
|
try:
|
|
if not self._client.bucket_exists(self.bucket):
|
|
self._log(f"Creating MinIO bucket '{self.bucket}'...")
|
|
self._client.make_bucket(self.bucket)
|
|
except Exception as e:
|
|
raise ArchiveUnavailable(
|
|
f"cannot reach MinIO at {endpoint}: {e} — set "
|
|
f"MINIO_ENDPOINT / MINIO_ACCESS_KEY / MINIO_SECRET_KEY"
|
|
) from e
|
|
|
|
def object_key(self, source_path: Path, input_dir: Path) -> str:
|
|
"""Bucket key mirrors the input tree's relative path."""
|
|
return source_path.relative_to(input_dir).as_posix()
|
|
|
|
def archive(self, source_path: Path, input_dir: Path) -> str:
|
|
"""Upload one source document; returns its s3:// URI."""
|
|
key = self.object_key(source_path, input_dir)
|
|
try:
|
|
self._client.fput_object(
|
|
self.bucket, key, str(source_path),
|
|
)
|
|
except Exception as e:
|
|
raise ArchiveUnavailable(
|
|
f"upload of '{key}' failed: {e}"
|
|
) from e
|
|
return f"s3://{self.bucket}/{key}"
|