"""Analyzer for weight field distribution.""" from typing import List, Dict, Any import logging from .base_analyzer import BaseAnalyzer logger = logging.getLogger(__name__) class WeightAnalyzer(BaseAnalyzer): """Analyze weight field distribution.""" def __init__(self, output_dir): super().__init__("weight", output_dir) def process_chunk(self, records: List[Dict[str, Any]]) -> Dict[str, Any]: """Process a chunk of records for weight analysis. Args: records: List of JSON records Returns: Intermediate results for this chunk """ weights = [] missing_weight_count = 0 weight_value_counts = {} for record in records: if "weight" in record: weight = record["weight"] weights.append(weight) # Count specific weight values weight_str = str(weight) if weight_str not in weight_value_counts: weight_value_counts[weight_str] = 0 weight_value_counts[weight_str] += 1 else: missing_weight_count += 1 return { "weights": weights, "weight_value_counts": weight_value_counts, "missing_weight_count": missing_weight_count, "total_records": len(records), } def aggregate(self, chunk_results: List[Dict[str, Any]]) -> Dict[str, Any]: """Aggregate results from all chunks. Args: chunk_results: List of results from each chunk Returns: Final aggregated analysis """ all_weights = [] all_value_counts = {} total_missing = 0 total_records = 0 for chunk in chunk_results: all_weights.extend(chunk.get("weights", [])) total_missing += chunk.get("missing_weight_count", 0) total_records += chunk.get("total_records", 0) # Merge value counts for value, count in chunk.get("weight_value_counts", {}).items(): if value not in all_value_counts: all_value_counts[value] = 0 all_value_counts[value] += count # Calculate distribution statistics weight_dist = self.calculate_distribution(all_weights) if all_weights else {} # Get unique weight values unique_weights = sorted(set(all_weights)) if all_weights else [] # Sort value counts sorted_value_counts = sorted(all_value_counts.items(), key=lambda x: x[1], reverse=True)[:20] # Create histogram weight_histogram = self.calculate_histogram(all_weights, bins=20) if all_weights else {} return { "summary": { "total_records": total_records, "records_with_weight": total_records - total_missing, "records_without_weight": total_missing, "unique_weight_values": len(unique_weights), }, "weight_distribution": weight_dist, "unique_weights": unique_weights[:100], # First 100 unique values "top_weight_values": sorted_value_counts, "weight_histogram": weight_histogram, }