"""Cache manager for storing and retrieving intermediate results.""" import json import pickle import hashlib from pathlib import Path from typing import Any, Optional import os import logging logger = logging.getLogger(__name__) class CacheManager: """Manage caching of analysis results and metadata.""" def __init__(self, cache_dir: Optional[Path] = None): """Initialize cache manager. Args: cache_dir: Directory for cache storage. If None, uses default paths. """ if cache_dir is None: # Try default paths in order for path in [ "~/data/suno_data_monitor/cache", "~/tmp/suno_data_monitor/cache", "/tmp/suno_data_monitor/cache", ]: expanded = Path(path).expanduser() if expanded.parent.exists(): cache_dir = expanded break else: cache_dir = Path("/tmp/suno_data_monitor/cache") self.cache_dir = Path(cache_dir).expanduser() self.cache_dir.mkdir(parents=True, exist_ok=True) logger.info(f"Cache directory: {self.cache_dir}") def _get_cache_key(self, file_path: Path, suffix: str = "") -> str: """Generate cache key based on file path and modification time. Args: file_path: Path to the file suffix: Additional suffix for the cache key Returns: Cache key string """ file_path = Path(file_path) if not file_path.exists(): raise FileNotFoundError(f"File not found: {file_path}") # Include file path, size, and modification time in key stat = file_path.stat() key_parts = [str(file_path.absolute()), str(stat.st_size), str(stat.st_mtime), suffix] key_str = "|".join(key_parts) # Create hash for filename hash_obj = hashlib.md5(key_str.encode()) return hash_obj.hexdigest() def get_line_count(self, file_path: Path) -> Optional[int]: """Get cached line count for a file. Args: file_path: Path to the file Returns: Line count if cached and valid, None otherwise """ try: cache_key = self._get_cache_key(file_path, "line_count") cache_file = self.cache_dir / f"{cache_key}.json" if cache_file.exists(): with open(cache_file, "r") as f: data = json.load(f) logger.info(f"Using cached line count for {file_path.name}: {data['line_count']:,}") return data["line_count"] except Exception as e: logger.warning(f"Failed to load cached line count: {e}") return None def save_line_count(self, file_path: Path, line_count: int) -> None: """Save line count to cache. Args: file_path: Path to the file line_count: Number of lines in the file """ try: cache_key = self._get_cache_key(file_path, "line_count") cache_file = self.cache_dir / f"{cache_key}.json" data = {"file_path": str(file_path.absolute()), "line_count": line_count} with open(cache_file, "w") as f: json.dump(data, f) logger.info(f"Cached line count for {file_path.name}: {line_count:,}") except Exception as e: logger.warning(f"Failed to cache line count: {e}") def get_ids(self, file_path: Path, dataset_type: str) -> Optional[set]: """Get cached ID set for a dataset. Args: file_path: Path to the dataset file dataset_type: Type of dataset (train/val/sft) Returns: Set of IDs if cached and valid, None otherwise """ try: cache_key = self._get_cache_key(file_path, f"ids_{dataset_type}") cache_file = self.cache_dir / f"{cache_key}.pkl" if cache_file.exists(): with open(cache_file, "rb") as f: ids = pickle.load(f) logger.info(f"Using cached IDs for {file_path.name}: {len(ids):,} IDs") return ids except Exception as e: logger.warning(f"Failed to load cached IDs: {e}") return None def save_ids(self, file_path: Path, dataset_type: str, ids: set) -> None: """Save ID set to cache. Args: file_path: Path to the dataset file dataset_type: Type of dataset (train/val/sft) ids: Set of IDs to cache """ try: cache_key = self._get_cache_key(file_path, f"ids_{dataset_type}") cache_file = self.cache_dir / f"{cache_key}.pkl" with open(cache_file, "wb") as f: pickle.dump(ids, f) logger.info(f"Cached {len(ids):,} IDs for {file_path.name}") except Exception as e: logger.warning(f"Failed to cache IDs: {e}") def get_analysis_result(self, file_path: Path, analyzer_name: str) -> Optional[Any]: """Get cached analysis result. Args: file_path: Path to the dataset file analyzer_name: Name of the analyzer Returns: Cached analysis result if valid, None otherwise """ try: cache_key = self._get_cache_key(file_path, f"analysis_{analyzer_name}") cache_file = self.cache_dir / f"{cache_key}.pkl" if cache_file.exists(): with open(cache_file, "rb") as f: result = pickle.load(f) logger.info(f"Using cached {analyzer_name} analysis for {file_path.name}") return result except Exception as e: logger.warning(f"Failed to load cached analysis: {e}") return None def save_analysis_result(self, file_path: Path, analyzer_name: str, result: Any) -> None: """Save analysis result to cache. Args: file_path: Path to the dataset file analyzer_name: Name of the analyzer result: Analysis result to cache """ try: cache_key = self._get_cache_key(file_path, f"analysis_{analyzer_name}") cache_file = self.cache_dir / f"{cache_key}.pkl" with open(cache_file, "wb") as f: pickle.dump(result, f) logger.info(f"Cached {analyzer_name} analysis for {file_path.name}") except Exception as e: logger.warning(f"Failed to cache analysis result: {e}") def clear_cache(self) -> int: """Clear all cache files. Returns: Number of files removed """ count = 0 for cache_file in self.cache_dir.glob("*"): if cache_file.is_file(): cache_file.unlink() count += 1 logger.info(f"Cleared {count} cache files") return count