"""Base analyzer class for all data analysis modules.""" from abc import ABC, abstractmethod from typing import List, Dict, Any import json import logging from pathlib import Path logger = logging.getLogger(__name__) class BaseAnalyzer(ABC): """Base class for all data analyzers.""" def __init__(self, name: str, output_dir: Path): """Initialize analyzer. Args: name: Name of the analyzer output_dir: Directory to save output files """ self.name = name self.output_dir = Path(output_dir) self.output_dir.mkdir(parents=True, exist_ok=True) @abstractmethod def process_chunk(self, records: List[Dict[str, Any]]) -> Dict[str, Any]: """Process a chunk of records. Args: records: List of JSON records from the dataset Returns: Intermediate results dictionary """ pass @abstractmethod def aggregate(self, chunk_results: List[Dict[str, Any]]) -> Dict[str, Any]: """Aggregate results from all chunks. Args: chunk_results: List of results from each chunk Returns: Final aggregated results """ pass def save_results(self, results: Dict[str, Any], suffix: str = "") -> Path: """Save results to JSON file. Args: results: Results dictionary to save suffix: Optional suffix for filename Returns: Path to saved file """ filename = f"{self.name}_analysis{suffix}.json" output_path = self.output_dir / filename # Remove non-JSON-serializable objects (like sets) clean_results = self._clean_for_json(results) with open(output_path, "w") as f: json.dump(clean_results, f, indent=2, ensure_ascii=False) logger.info(f"Saved {self.name} results to {output_path}") return output_path def _clean_for_json(self, obj: Any) -> Any: """Clean object for JSON serialization. Args: obj: Object to clean Returns: JSON-serializable object """ if isinstance(obj, dict): return {k: self._clean_for_json(v) for k, v in obj.items() if k != "id_set"} elif isinstance(obj, list): return [self._clean_for_json(item) for item in obj] elif isinstance(obj, set): return list(obj)[:1000] # Convert sets to lists, limit size else: return obj def calculate_distribution(self, values: List[float]) -> Dict[str, Any]: """Calculate distribution statistics for numeric values. Args: values: List of numeric values Returns: Dictionary with distribution statistics """ if not values: return {} sorted_values = sorted(values) n = len(sorted_values) return { "count": n, "min": sorted_values[0], "max": sorted_values[-1], "mean": sum(sorted_values) / n, "median": sorted_values[n // 2], "p25": sorted_values[n // 4] if n >= 4 else sorted_values[0], "p75": sorted_values[3 * n // 4] if n >= 4 else sorted_values[-1], "p95": sorted_values[int(0.95 * n)] if n >= 20 else sorted_values[-1], "p99": sorted_values[int(0.99 * n)] if n >= 100 else sorted_values[-1], } def calculate_histogram(self, values: List[float], bins: int = 20) -> Dict[str, Any]: """Calculate histogram for numeric values. Args: values: List of numeric values bins: Number of bins for histogram Returns: Dictionary with histogram data """ if not values: return {} min_val = min(values) max_val = max(values) if min_val == max_val: return {"bins": [min_val], "counts": [len(values)]} bin_width = (max_val - min_val) / bins bin_edges = [min_val + i * bin_width for i in range(bins + 1)] bin_counts = [0] * bins for value in values: bin_idx = min(int((value - min_val) / bin_width), bins - 1) bin_counts[bin_idx] += 1 return {"bin_edges": bin_edges, "bin_counts": bin_counts, "bin_width": bin_width}