"""Analyzer for text field including bracket patterns and special tags.""" import re from collections import defaultdict, Counter from typing import List, Dict, Any, Tuple import logging from .base_analyzer import BaseAnalyzer from ..utils.stats_utils import aggregate_counters, merge_distributions logger = logging.getLogger(__name__) class TextAnalyzer(BaseAnalyzer): """Analyze text field for patterns, brackets, and special formatting.""" def __init__(self, output_dir): super().__init__("text", output_dir) # Regex patterns for different bracket types self.bracket_patterns = { "square": r"\[([^\]]*)\]", "curly": r"\{([^\}]*)\}", "angle": r"<([^>]*)>", "parentheses": r"\(([^\)]*)\)", "chinese_angle": r"【([^】]*)】", "chinese_book": r"《([^》]*)》", "japanese_corner": r"「([^」]*)」", "japanese_double": r"『([^』]*)』", } def process_chunk(self, records: List[Dict[str, Any]]) -> Dict[str, Any]: """Process a chunk of records for text analysis. Args: records: List of JSON records Returns: Intermediate results for this chunk """ text_lengths = [] bracket_content = defaultdict(list) bracket_positions = defaultdict(list) bracket_counts = defaultdict(int) texts_with_brackets = defaultdict(int) missing_text_count = 0 empty_text_count = 0 for record in records: text = record.get("text", "") if "text" not in record: missing_text_count += 1 continue if not text: empty_text_count += 1 continue text_lengths.append(len(text)) # Analyze each bracket type has_any_bracket = False for bracket_type, pattern in self.bracket_patterns.items(): matches = re.findall(pattern, text) if matches: has_any_bracket = True texts_with_brackets[bracket_type] += 1 bracket_counts[bracket_type] += len(matches) # Store content and lengths for match in matches: bracket_content[bracket_type].append(len(match)) # Find positions (start, middle, end) for match in re.finditer(pattern, text): pos = match.start() relative_pos = pos / len(text) if relative_pos < 0.1: position = "start" elif relative_pos > 0.9: position = "end" else: position = "middle" if f"{bracket_type}_{position}" not in bracket_positions: bracket_positions[f"{bracket_type}_{position}"] = 0 bracket_positions[f"{bracket_type}_{position}"] += 1 # Special patterns in text if has_any_bracket: texts_with_brackets["any"] += 1 return { "text_lengths": text_lengths, "bracket_content_lengths": dict(bracket_content), "bracket_positions": dict(bracket_positions), "bracket_counts": dict(bracket_counts), "texts_with_brackets": dict(texts_with_brackets), "missing_text_count": missing_text_count, "empty_text_count": empty_text_count, "total_records": len(records), } def aggregate(self, chunk_results: List[Dict[str, Any]]) -> Dict[str, Any]: """Aggregate results from all chunks. Args: chunk_results: List of results from each chunk Returns: Final aggregated analysis """ all_text_lengths = [] all_bracket_content = defaultdict(list) all_bracket_positions = [] all_bracket_counts = [] all_texts_with_brackets = [] total_missing = 0 total_empty = 0 total_records = 0 for chunk in chunk_results: all_text_lengths.extend(chunk.get("text_lengths", [])) # Merge bracket content lengths for bracket_type, lengths in chunk.get("bracket_content_lengths", {}).items(): all_bracket_content[bracket_type].extend(lengths) all_bracket_positions.append(chunk.get("bracket_positions", {})) all_bracket_counts.append(chunk.get("bracket_counts", {})) all_texts_with_brackets.append(chunk.get("texts_with_brackets", {})) total_missing += chunk.get("missing_text_count", 0) total_empty += chunk.get("empty_text_count", 0) total_records += chunk.get("total_records", 0) # Aggregate counters total_bracket_positions = aggregate_counters(all_bracket_positions) total_bracket_counts = aggregate_counters(all_bracket_counts) total_texts_with_brackets = aggregate_counters(all_texts_with_brackets) # Calculate distributions text_length_dist = self.calculate_distribution(all_text_lengths) if all_text_lengths else {} # Bracket content length distributions bracket_content_stats = {} for bracket_type, lengths in all_bracket_content.items(): if lengths: bracket_content_stats[bracket_type] = self.calculate_distribution(lengths) # Text length histogram (binned) if all_text_lengths: length_bins = [0, 100, 500, 1000, 2000, 5000, 10000, 20000, 50000, float("inf")] length_histogram = defaultdict(int) for length in all_text_lengths: for i in range(len(length_bins) - 1): if length_bins[i] <= length < length_bins[i + 1]: bin_label = ( f"{length_bins[i]}-{length_bins[i+1]}" if length_bins[i + 1] != float("inf") else f"{length_bins[i]}+" ) length_histogram[bin_label] += 1 break else: length_histogram = {} # Position analysis position_stats = {} for key in ["start", "middle", "end"]: position_sum = sum( total_bracket_positions.get(f"{bt}_{key}", 0) for bt in self.bracket_patterns.keys() ) if position_sum > 0: position_stats[key] = position_sum return { "summary": { "total_records": total_records, "records_with_text": total_records - total_missing, "records_without_text": total_missing, "empty_texts": total_empty, "non_empty_texts": len(all_text_lengths), "texts_with_any_brackets": total_texts_with_brackets.get("any", 0), }, "text_length_distribution": text_length_dist, "text_length_histogram": dict(length_histogram), "bracket_statistics": { "counts_by_type": total_bracket_counts, "texts_with_brackets_by_type": total_texts_with_brackets, "position_distribution": total_bracket_positions, "position_summary": position_stats, "content_length_stats": bracket_content_stats, }, "discovered_bracket_types": sorted( [bt for bt, count in total_bracket_counts.items() if count > 0] ), }