165 lines
4.8 KiB
Python
165 lines
4.8 KiB
Python
"""
|
|
Btrfs content-addressable storage with CoW reflink.
|
|
|
|
Uses `cp --reflink=always` to create copy-on-write reflinks on btrfs
|
|
filesystems (instant copy, zero extra disk usage until divergence).
|
|
|
|
Falls back to:
|
|
1. `cp --reflink=auto` (let filesystem decide — works on btrfs, falls
|
|
back to normal copy on ext4/xfs without error)
|
|
2. `shutil.copy2` (preserves metadata)
|
|
3. `shutil.copy` (last resort)
|
|
|
|
Also exposes `is_btrfs(path)` to detect whether a path is on btrfs.
|
|
"""
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
|
|
def is_btrfs(path) -> bool:
|
|
"""Return True if `path` lives on a btrfs filesystem."""
|
|
try:
|
|
out = subprocess.check_output(
|
|
["stat", "-f", "-c", "%T", str(path)],
|
|
stderr=subprocess.DEVNULL,
|
|
).decode().strip()
|
|
return out.lower() == "btrfs"
|
|
except (subprocess.CalledProcessError, FileNotFoundError, OSError):
|
|
return False
|
|
|
|
|
|
def store_reflink(src, dst) -> bool:
|
|
"""
|
|
Copy `src` to `dst` using CoW reflink when possible.
|
|
|
|
Returns True if a true reflink was created, False if a regular copy
|
|
was used as fallback.
|
|
"""
|
|
src = str(src)
|
|
dst = str(dst)
|
|
|
|
# Ensure destination directory exists
|
|
Path(dst).parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Try true reflink first (btrfs only)
|
|
try:
|
|
result = subprocess.run(
|
|
["cp", "--reflink=always", src, dst],
|
|
capture_output=True,
|
|
)
|
|
if result.returncode == 0:
|
|
return True
|
|
# If --reflink=always fails, the source/dest aren't on the same btrfs
|
|
except FileNotFoundError:
|
|
pass # cp not available (unlikely on Linux)
|
|
|
|
# Fall back to auto-reflink (works on btrfs without error, regular copy elsewhere)
|
|
try:
|
|
subprocess.run(
|
|
["cp", "--reflink=auto", src, dst],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
# Preserve metadata
|
|
shutil.copystat(src, dst)
|
|
return False
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
pass
|
|
|
|
# Last resort: pure Python copy
|
|
try:
|
|
shutil.copy2(src, dst)
|
|
except Exception:
|
|
shutil.copy(src, dst)
|
|
return False
|
|
|
|
|
|
def reflink_snapshot(src_dir, dst_dir) -> dict:
|
|
"""
|
|
Snapshot an entire directory tree using CoW reflinks.
|
|
|
|
Returns a dict with:
|
|
- reflinked: count of files that got true reflinks
|
|
- copied: count of files that fell back to regular copy
|
|
- failed: count of files that couldn't be copied
|
|
- total_bytes: total size of files snapshotted
|
|
"""
|
|
src = Path(src_dir)
|
|
dst = Path(dst_dir)
|
|
if not src.is_dir():
|
|
return {"reflinked": 0, "copied": 0, "failed": 0, "total_bytes": 0, "error": "src not a dir"}
|
|
|
|
dst.mkdir(parents=True, exist_ok=True)
|
|
reflinked = copied = failed = 0
|
|
total_bytes = 0
|
|
|
|
for root, dirs, files in os.walk(src):
|
|
rel = Path(root).relative_to(src)
|
|
(dst / rel).mkdir(parents=True, exist_ok=True)
|
|
for f in files:
|
|
src_file = Path(root) / f
|
|
dst_file = dst / rel / f
|
|
try:
|
|
size = src_file.stat().st_size
|
|
if store_reflink(src_file, dst_file):
|
|
reflinked += 1
|
|
else:
|
|
copied += 1
|
|
total_bytes += size
|
|
except Exception:
|
|
failed += 1
|
|
|
|
return {
|
|
"reflinked": reflinked,
|
|
"copied": copied,
|
|
"failed": failed,
|
|
"total_bytes": total_bytes,
|
|
}
|
|
|
|
|
|
def cas_path(config, key) -> str:
|
|
"""Resolve the CAS storage path for a given key."""
|
|
base = Path(config.get("btrfs_path", "/var/lib/fester/cas"))
|
|
path = base / key
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
return str(path)
|
|
|
|
|
|
def cas_store(config, key, src_path) -> dict:
|
|
"""Store a file/directory in the CAS under `key` using reflink.
|
|
Returns the snapshot stats."""
|
|
dst = cas_path(config, key)
|
|
src = Path(src_path)
|
|
if src.is_dir():
|
|
return reflink_snapshot(src, dst)
|
|
else:
|
|
reflinked = store_reflink(src, dst)
|
|
return {
|
|
"reflinked": 1 if reflinked else 0,
|
|
"copied": 0 if reflinked else 1,
|
|
"failed": 0,
|
|
"total_bytes": src.stat().st_size if src.exists() else 0,
|
|
"dst": dst,
|
|
}
|
|
|
|
|
|
def cas_exists(config, key) -> bool:
|
|
"""Check if a CAS entry exists."""
|
|
return Path(cas_path(config, key)).exists()
|
|
|
|
|
|
def cas_retrieve(config, key, dst_path) -> bool:
|
|
"""Retrieve a CAS entry back to `dst_path` (reflink copy)."""
|
|
src = Path(cas_path(config, key))
|
|
if not src.exists():
|
|
return False
|
|
if src.is_dir():
|
|
stats = reflink_snapshot(src, dst_path)
|
|
return stats["failed"] == 0
|
|
else:
|
|
return store_reflink(src, dst_path)
|