#!/usr/bin/env python3 """ Plot evaluation scores from score files. This script automatically discovers score files in input directories, groups them by evaluation type, and creates plots with mean + std dev error bars. Supports both grouped and ungrouped evaluations, and integrates baseline scores. """ import argparse import glob import json import os import re from collections import defaultdict from dataclasses import dataclass from pathlib import Path from typing import Dict, List, Optional, Tuple, Union, Any import matplotlib.pyplot as plt import numpy as np import pandas as pd from matplotlib.backends.backend_pdf import PdfPages # Color palette for consistent plotting COLORS = [ (0.12, 0.47, 0.71), # Steel blue (0.17, 0.63, 0.17), # Forest green (1.00, 0.50, 0.00), # Orange (0.54, 0.17, 0.89), # Blue violet (0.00, 0.75, 1.00), # Deep sky blue (1.00, 0.40, 0.00), # Bright orange (0.00, 0.50, 0.50), # Teal (1.00, 0.00, 1.00), # Bright magenta (0.85, 0.65, 0.13), # Golden (0.00, 0.80, 0.80), # Turquoise (0.84, 0.15, 0.16), # Crimson red (0.20, 0.20, 0.80), # Navy blue (0.60, 0.80, 0.20), # Olive green (0.80, 0.20, 0.20), # Red (0.40, 0.80, 0.60), # Sea green (0.80, 0.40, 0.00), # Burnt sienna ] @dataclass class ScoreData: """Container for score data from a single file.""" model_name: str eval_type: str file_path: str raw_data: Dict[str, Any] scores: Dict[str, List[float]] # group_name -> list of scores is_grouped: bool metadata: Dict[str, Any] = None @dataclass class PlotConfig: """Configuration for plotting.""" title: str ylabel: str higher_is_better: bool = True ylim: Optional[Tuple[float, float]] = None transform_scores: Optional[str] = None # 'diversity' for 1-similarity class ScoreLoader: """Loads and processes score files.""" def __init__(self, baseline_config_path: Optional[str] = None): self.baseline_config = self._load_baseline_config(baseline_config_path) def _load_baseline_config(self, config_path: Optional[str]) -> Dict: """Load baseline configuration from JSON file.""" if not config_path or not os.path.exists(config_path): return {} try: with open(config_path, "r") as f: return json.load(f) except Exception as e: print(f"Warning: Could not load baseline config from {config_path}: {e}") return {} def discover_score_files(self, input_dirs: List[str]) -> List[str]: """Discover all .npz score files in input directories.""" score_files = [] for input_dir in input_dirs: if not os.path.exists(input_dir): print(f"Warning: Directory {input_dir} does not exist") continue for filename in os.listdir(input_dir): if filename.endswith(".npz"): filepath = os.path.join(input_dir, filename) if os.path.isfile(filepath): score_files.append(filepath) return sorted(score_files) def extract_model_name(self, filepath: str) -> str: """Extract model name from filename and truncate to first 10 characters.""" filename = os.path.basename(filepath).replace(".npz", "") parts = filename.split("_") # Try to find feat_eval position for new format try: feat_eval_idx = parts.index("feat_eval") # Look for timestamp at the end (format: YYYY_MM_DD-HH_MM_SS) timestamp_idx = None for i in range(len(parts) - 1, -1, -1): if len(parts[i]) == 4 and parts[i].isdigit(): # Year if i + 4 < len(parts) and "-" in parts[i + 2]: timestamp_idx = i break if timestamp_idx is None: model_name = parts[-1] else: model_name = parts[timestamp_idx - 1] # Extract extra_tags (everything before test_type) extra_tags = "_".join(parts[: feat_eval_idx - 1]) full_name = f"{extra_tags}_{model_name}" if extra_tags else model_name except ValueError: # Fallback: use filename as model name full_name = filename # Truncate to first 10 characters for better readability return full_name[:10] def detect_eval_type(self, filepath: str) -> str: """Detect evaluation type from filename. Patterns match the baseline_config.json eval_type_patterns for consistency. """ filename = os.path.basename(filepath).lower() # Map filename patterns to evaluation types (ordered by specificity - most specific first!) # Pattern: *lyrics*feat_eval_hoot_cer* if "lyrics" in filename and "feat_eval_hoot_cer" in filename: return "hoot_cer" # Pattern: *lyrics*feat_eval_cer* (but not hoot_cer which was already matched) elif "lyrics" in filename and "feat_eval_cer" in filename: return "cer" # Pattern: *infill*whisper* elif "infill" in filename and "whisper" in filename: return "infill" # Pattern: *lyrics*whisper* elif "lyrics" in filename and "whisper" in filename: return "lyrics" # Pattern: *div_base_gen*self_sim* elif "div" in filename and "base_gen" in filename and "self_sim" in filename: return "diversity" # Pattern: *inst*drift*artist_sim* elif "inst" in filename and "drift" in filename and "artist_sim" in filename: return "inst_drift" # Pattern: *div_genre*intro* elif "div" in filename and "genre" in filename and "intro" in filename: return "intro" # Pattern: *div_genre*duration* elif "div" in filename and "genre" in filename and "duration" in filename: return "duration" # Pattern: *subg*within_parent* elif "subg" in filename and "within_parent" in filename: return "subgenre_within" # Pattern: *subg*with_parent* elif "subg" in filename and "with_parent" in filename: return "subgenre_between" # Pattern: *ag-cov*cover*self_sim* elif "ag-cov" in filename and "cover" in filename and "self_sim" in filename: return "cover" # Pattern: *ag-art*artist*vox_sim* elif "ag-art" in filename and "artist" in filename and "vox_sim" in filename: return "artist_vox" # Pattern: *ag-art*artist*self_sim* elif "ag-art" in filename and "artist" in filename and "self_sim" in filename: return "artist_self" # Pattern: *hard_artist*self_sim* elif "hard" in filename and "artist" in filename and "self_sim" in filename: return "artist_hard" else: return "unknown" def load_score_file(self, filepath: str) -> Optional[ScoreData]: """Load a single score file and extract relevant data.""" try: data = np.load(filepath, allow_pickle=True) model_name = self.extract_model_name(filepath) eval_type = self.detect_eval_type(filepath) # Convert numpy data to dict for easier handling raw_data = {key: data[key] for key in data.keys()} # Extract scores based on evaluation type scores, is_grouped = self._extract_scores(raw_data, eval_type) # Debug: print warning if scores are empty if not scores or all(not v for v in scores.values()): print( f"Warning: No scores extracted from {os.path.basename(filepath)} (type: {eval_type}, keys: {list(raw_data.keys())})" ) return ScoreData( model_name=model_name, eval_type=eval_type, file_path=filepath, raw_data=raw_data, scores=scores, is_grouped=is_grouped, metadata=raw_data.get("metadata", {}), ) except Exception as e: print(f"Error loading {filepath}: {e}") return None def _extract_scores(self, raw_data: Dict, eval_type: str) -> Tuple[Dict[str, List[float]], bool]: """Extract scores from raw data based on evaluation type.""" scores = {} is_grouped = False if eval_type == "diversity": # Diversity scores: convert similarity to diversity (1 - similarity) if "main" in raw_data: df = pd.DataFrame(list(raw_data["main"])) if "genre" in df.columns: is_grouped = True for genre, group in df.groupby("genre"): diversity_scores = 1 - group["similarity"].astype(float) scores[genre] = diversity_scores.tolist() else: scores["all"] = (1 - df["similarity"].astype(float)).tolist() elif eval_type == "inst_drift": # Instrumental drift can be grouped by instrument category # Skip metadata key if present if isinstance(raw_data, dict): for group, group_scores in raw_data.items(): if group == "metadata": # Skip metadata continue # Handle numpy arrays (including 0-d arrays) if isinstance(group_scores, np.ndarray): if group_scores.ndim == 0: # Scalar scores[group] = [float(group_scores)] else: scores[group] = group_scores.tolist() elif isinstance(group_scores, list): scores[group] = group_scores elif hasattr(group_scores, "__iter__"): scores[group] = list(group_scores) else: scores[group] = [float(group_scores)] # Mark as grouped if we have more than just "all" if len(scores) > 1 or (len(scores) == 1 and "all" not in scores): is_grouped = True elif eval_type == "intro": if "intro" in raw_data: intro_data = raw_data["intro"] # New format: intro is always a list of scores scores["all"] = ( list(intro_data) if hasattr(intro_data, "__iter__") else [float(intro_data)] ) elif eval_type == "hoot_cer": if "hoot_cer" in raw_data: cer_data = raw_data["hoot_cer"] # Handle grouped hoot_cer data (new format: dict with group keys) if isinstance(cer_data, dict) or ( hasattr(cer_data, "item") and isinstance(cer_data.item(), dict) ): if hasattr(cer_data, "item"): cer_data = cer_data.item() for group_key, group_value in cer_data.items(): if group_key != "metadata": if isinstance(group_value, np.ndarray): scores[group_key] = group_value.tolist() elif hasattr(group_value, "__iter__") and not isinstance( group_value, (str, dict) ): scores[group_key] = list(group_value) else: scores[group_key] = [float(group_value)] if group_key != "all": is_grouped = True # Handle flat list (old format) else: scores["all"] = ( list(cer_data) if hasattr(cer_data, "__iter__") else [float(cer_data)] ) elif eval_type == "duration": if "duration" in raw_data: duration_data = raw_data["duration"] # New format: duration is always a list of scores scores["all"] = ( list(duration_data) if hasattr(duration_data, "__iter__") else [float(duration_data)] ) elif eval_type in ["subgenre_within", "subgenre_between"]: # Store subgenre scores for later difference calculation key = "within_genre_pairs" if eval_type == "subgenre_within" else "between_genre_pairs" if key in raw_data: genre_data = raw_data[key] if isinstance(genre_data, dict) or hasattr(genre_data, "item"): if hasattr(genre_data, "item"): genre_data = genre_data.item() is_grouped = True for genre, genre_scores in genre_data.items(): scores[genre] = ( list(genre_scores) if hasattr(genre_scores, "__iter__") else [float(genre_scores)] ) elif eval_type in ["cover", "artist_vox", "artist_self", "artist_hard"]: # Similarity-based evaluations # Remove metadata keys and extract score arrays score_keys = [k for k in raw_data.keys() if k not in ["metadata", "evaluation_type"]] all_scores = [] for key in score_keys: score_data = raw_data[key] if isinstance(score_data, np.ndarray) and len(score_data.shape) > 0: all_scores.extend(score_data.flatten()) elif hasattr(score_data, "__iter__"): all_scores.extend(score_data) scores["all"] = all_scores elif eval_type in ["infill", "lyrics"]: # Infill/Lyrics WER scores # Check for data in "main" key first (new format), then fall back to direct keys (old format) main_data = raw_data.get("main") if main_data is not None: # New format: data is in raw_data["main"][group_name] if hasattr(main_data, "item"): main_data = main_data.item() if isinstance(main_data, dict): for group_key, group_value in main_data.items(): if group_key != "metadata": if hasattr(group_value, "item"): group_value = group_value.item() if isinstance(group_value, dict) and "wers" in group_value: scores[group_key] = group_value["wers"] if group_key != "all": is_grouped = True else: # Old format: data is directly in raw_data[group_name] if "all" in raw_data: all_data = raw_data["all"] if hasattr(all_data, "item"): all_data = all_data.item() if isinstance(all_data, dict) and "wers" in all_data: scores["all"] = all_data["wers"] # Check for grouped data for key, value in raw_data.items(): if key != "all" and key != "metadata": if hasattr(value, "item"): value = value.item() if isinstance(value, dict) and "wers" in value: scores[key] = value["wers"] is_grouped = True elif eval_type == "cer": # CER scores - can be grouped by genre like WER if "cer" in raw_data: cer_data = raw_data["cer"] # Handle grouped CER data (new format: dict with group keys) if isinstance(cer_data, dict) or ( hasattr(cer_data, "item") and isinstance(cer_data.item(), dict) ): if hasattr(cer_data, "item"): cer_data = cer_data.item() for group_key, group_value in cer_data.items(): if group_key != "metadata": if isinstance(group_value, np.ndarray): scores[group_key] = group_value.tolist() elif hasattr(group_value, "__iter__") and not isinstance( group_value, (str, dict) ): scores[group_key] = list(group_value) else: scores[group_key] = [float(group_value)] if group_key != "all": is_grouped = True # Handle flat list (old format) elif isinstance(cer_data, np.ndarray): scores["all"] = cer_data.tolist() elif hasattr(cer_data, "__iter__") and not isinstance(cer_data, (str, dict)): scores["all"] = list(cer_data) else: scores["all"] = [float(cer_data)] return scores, is_grouped def load_baseline_scores(self, eval_type: str) -> Dict[str, ScoreData]: """Load baseline scores for a given evaluation type.""" baselines = {} if not self.baseline_config: return baselines # Support both old format (evaluations) and new format (models + patterns) if "evaluations" in self.baseline_config: # Old format - direct file mapping eval_baselines = self.baseline_config["evaluations"].get(eval_type, {}) baselines_path = self.baseline_config.get("baselines_path", "") for model_name, filename in eval_baselines.items(): filepath = os.path.join(baselines_path, filename) if os.path.exists(filepath): baseline_data = self.load_score_file(filepath) if baseline_data: baseline_data.model_name = model_name # Override with baseline name baselines[model_name] = baseline_data elif "models" in self.baseline_config and "eval_type_patterns" in self.baseline_config: # New format - folder-based discovery models = self.baseline_config["models"] patterns = self.baseline_config["eval_type_patterns"].get(eval_type, []) baselines_path = self.baseline_config.get("baselines_path", "") for model_name, model_folder in models.items(): model_path = os.path.join(baselines_path, model_folder) if not os.path.exists(model_path): print(f"Warning: Model folder {model_path} does not exist") continue # Find files matching any of the patterns for this eval type matching_files = [] for pattern in patterns: search_pattern = os.path.join(model_path, f"{pattern}.npz") matching_files.extend(glob.glob(search_pattern)) # Use the first matching file (could be enhanced to handle multiple matches) if matching_files: filepath = matching_files[0] # Take first match baseline_data = self.load_score_file(filepath) if baseline_data: baseline_data.model_name = model_name # Override with baseline name baselines[model_name] = baseline_data print( f"Loaded baseline: {model_name} ({eval_type}) from {os.path.basename(filepath)}" ) else: print( f"Warning: No files found for {model_name} ({eval_type}) with patterns {patterns}" ) return baselines def combine_subgenre_scores(self, eval_groups: Dict) -> Dict: """Combine subgenre within and between scores into difference scores.""" if "subgenre_within" not in eval_groups or "subgenre_between" not in eval_groups: return eval_groups within_data_list = eval_groups["subgenre_within"] between_data_list = eval_groups["subgenre_between"] # Create a mapping of model names to their data within_by_model = {data.model_name: data for data in within_data_list} between_by_model = {data.model_name: data for data in between_data_list} # Find models that have both within and between data common_models = set(within_by_model.keys()) & set(between_by_model.keys()) subgenre_diff_data = [] for model_name in common_models: within_data = within_by_model[model_name] between_data = between_by_model[model_name] # Calculate differences for each genre diff_scores = {} all_genres = set(within_data.scores.keys()) & set(between_data.scores.keys()) for genre in all_genres: within_scores = within_data.scores[genre] between_scores = between_data.scores[genre] if within_scores and between_scores: # Calculate statistics for both within and between within_array = np.array(within_scores) between_array = np.array(between_scores) within_mean = np.mean(within_array) between_mean = np.mean(between_array) within_std = np.std(within_array) between_std = np.std(between_array) # Calculate multiple difference samples for error estimation # Use bootstrap resampling to get distribution of differences n_bootstrap = 1000 diff_samples = [] for _ in range(n_bootstrap): # Resample with replacement within_sample = np.random.choice( within_array, size=len(within_array), replace=True ) between_sample = np.random.choice( between_array, size=len(between_array), replace=True ) # Calculate difference of means diff_samples.append(np.mean(within_sample) - np.mean(between_sample)) # Store the bootstrap samples as our "scores" for this genre diff_scores[genre] = diff_samples if diff_scores: # Create new ScoreData for the difference diff_score_data = ScoreData( model_name=model_name, eval_type="subgenre_diff", file_path=within_data.file_path, # Use within file path as reference raw_data={}, scores=diff_scores, is_grouped=True, metadata={"description": "Within - Between subgenre similarity difference"}, ) subgenre_diff_data.append(diff_score_data) # Remove the separate within/between and add the combined difference new_eval_groups = eval_groups.copy() if "subgenre_within" in new_eval_groups: del new_eval_groups["subgenre_within"] if "subgenre_between" in new_eval_groups: del new_eval_groups["subgenre_between"] if subgenre_diff_data: new_eval_groups["subgenre_diff"] = subgenre_diff_data return new_eval_groups class PlotGenerator: """Generates plots from score data.""" def __init__(self): self.plot_configs = { "diversity": PlotConfig( title="Diversity Across Groups", ylabel="Mean Prompt Diversity", higher_is_better=True, ylim=(0.0, 0.5), ), "inst_drift": PlotConfig( title="Instrumental Drift (Early vs Late)", ylabel="Cosine Similarity (Higher = Less Drift)", higher_is_better=True, ylim=(0.0, 1.0), ), "intro": PlotConfig( title="Introduction Length Distribution", ylabel="Introduction Length (seconds)", higher_is_better=False, # Shorter intros might be better for some use cases ylim=None, # Let it auto-scale ), "duration": PlotConfig( title="Audio Duration Distribution", ylabel="Duration (seconds)", higher_is_better=False, # Neutral metric ylim=None, # Let it auto-scale ), "subgenre_diff": PlotConfig( title="Subgenre Specificity (Within - Between Similarity)", ylabel="Similarity Difference", higher_is_better=True, ), "cover": PlotConfig( title="Cover Self Similarity", ylabel="Mean Similarity", higher_is_better=True ), "artist_vox": PlotConfig( title="Artist Vocal Similarity", ylabel="Mean Similarity", higher_is_better=True ), "artist_self": PlotConfig( title="Artist Self Similarity", ylabel="Mean Similarity", higher_is_better=True ), "artist_hard": PlotConfig( title="Artist Self Similarity (Hard)", ylabel="Mean Similarity", higher_is_better=True ), "infill": PlotConfig( title="Infill WER Scores", ylabel="Mean WER ± SEM", # Standard Error of Mean higher_is_better=False, ), "lyrics": PlotConfig( title="Lyrics WER Scores", ylabel="Mean WER ± SEM", # Standard Error of Mean higher_is_better=False, ), "cer": PlotConfig( title="Character Error Rate (CER)", ylabel="Mean CER ± SEM", # Standard Error of Mean higher_is_better=False, ), "hoot_cer": PlotConfig( title="Hoot Character Error Rate", ylabel="Mean Hoot CER ± SEM", # Standard Error of Mean higher_is_better=False, ), } def create_plot( self, score_data_list: List[ScoreData], eval_type: str ) -> Tuple[plt.Figure, plt.Axes]: """Create a plot for the given evaluation type and score data.""" config = self.plot_configs.get( eval_type, PlotConfig(title=f"{eval_type} Scores", ylabel="Score") ) # Check if any data is grouped is_grouped = any(data.is_grouped for data in score_data_list) if is_grouped: return self._create_grouped_plot(score_data_list, config) else: return self._create_ungrouped_plot(score_data_list, config) def _create_grouped_plot( self, score_data_list: List[ScoreData], config: PlotConfig ) -> Tuple[plt.Figure, plt.Axes]: """Create a grouped bar plot.""" fig, ax = plt.subplots(figsize=(12, 6)) # Collect all groups across all models all_groups = set() for data in score_data_list: all_groups.update(data.scores.keys()) groups = sorted(list(all_groups)) models = [data.model_name for data in score_data_list] # Calculate positions x = np.arange(len(groups)) bar_width = 0.8 / len(models) # Plot bars for each model for i, data in enumerate(score_data_list): means = [] stds = [] for group in groups: if group in data.scores and data.scores[group]: scores = np.array(data.scores[group]) scores = scores[~np.isnan(scores)] # Remove NaN values # Use appropriate error bars based on evaluation type if data.eval_type in ["infill", "lyrics", "hoot_cer", "cer"]: # For WER/CER, use standard error of the mean mean = np.mean(scores) std = np.std(scores) / np.sqrt(len(scores)) # Standard Error of Mean elif data.eval_type == "subgenre_diff": # For subgenre_diff, scores are already bootstrap samples # Use standard deviation to show the bootstrap distribution spread mean = np.mean(scores) std = np.std(scores) else: # Use bootstrap confidence interval for all other metrics mean, std = self._bootstrap_confidence_interval(scores) means.append(mean) stds.append(std) else: means.append(0) stds.append(0) offset = i - (len(models) - 1) / 2 color = COLORS[i % len(COLORS)] ax.bar( x + offset * bar_width, means, yerr=stds, width=bar_width, label=data.model_name, color=color, alpha=0.8, capsize=3, ) # Customize plot ax.set_xticks(x) ax.set_xticklabels(groups, rotation=45, ha="right") ax.set_ylabel(config.ylabel) ax.set_title(config.title) ax.legend() ax.grid(True, axis="y", linestyle="--", alpha=0.3) if config.ylim: ax.set_ylim(config.ylim) plt.tight_layout() return fig, ax def _create_ungrouped_plot( self, score_data_list: List[ScoreData], config: PlotConfig ) -> Tuple[plt.Figure, plt.Axes]: """Create an ungrouped plot - box plot for similarity scores and distribution metrics, bar plot for others.""" eval_type = score_data_list[0].eval_type if score_data_list else "unknown" # Use box plots for similarity-based evaluations and distribution metrics if eval_type in [ "artist_hard", "artist_self", "artist_vox", "cover", "inst_drift", "intro", "duration", "hoot_cer", "cer", ]: return self._create_box_plot(score_data_list, config) else: return self._create_bar_plot(score_data_list, config) def _create_box_plot( self, score_data_list: List[ScoreData], config: PlotConfig ) -> Tuple[plt.Figure, plt.Axes]: """Create a box and whisker plot for similarity scores and distribution metrics.""" fig, ax = plt.subplots(figsize=(12, 6)) models = [data.model_name for data in score_data_list] all_scores_list = [] for data in score_data_list: # Combine all scores from all groups all_scores = [] for group_scores in data.scores.values(): all_scores.extend(group_scores) if all_scores: scores = np.array(all_scores, dtype=float) scores = scores[~np.isnan(scores)] # Remove NaN values all_scores_list.append(scores) else: all_scores_list.append(np.array([])) # Create box plot bp = ax.boxplot( all_scores_list, tick_labels=models, patch_artist=True, notch=True, showfliers=False ) # Turn off default outliers, we'll plot all points # Color the boxes for i, patch in enumerate(bp["boxes"]): patch.set_facecolor(COLORS[i % len(COLORS)]) patch.set_alpha(0.7) # Make median lines (box centers) black for better visibility for median in bp["medians"]: median.set_color("black") # Add individual points with jitter (skip for metrics with many samples to avoid clutter) eval_type = score_data_list[0].eval_type if score_data_list else "unknown" if eval_type not in ["inst_drift", "intro", "duration", "hoot_cer", "cer"]: for i, scores in enumerate(all_scores_list): if len(scores) > 0: # Create jittered x positions x_pos = i + 1 # Box plot positions are 1-indexed jitter_strength = 0.1 # Keep points within box width # Create random jitter around the box position np.random.seed(42 + i) # Consistent jitter for reproducibility x_jittered = x_pos + np.random.uniform( -jitter_strength, jitter_strength, len(scores) ) # Plot individual points ax.scatter( x_jittered, scores, alpha=0.6, s=20, # Point size color="black", edgecolors="white", linewidth=0.5, zorder=3, ) # Ensure points are on top # Customize plot ax.set_xticklabels(models, rotation=45, ha="right") ax.set_ylabel(config.ylabel) ax.set_title(config.title) ax.grid(True, axis="y", linestyle="--", alpha=0.3) if config.ylim: ax.set_ylim(config.ylim) plt.tight_layout() return fig, ax def _create_bar_plot( self, score_data_list: List[ScoreData], config: PlotConfig ) -> Tuple[plt.Figure, plt.Axes]: """Create a bar plot with error bars.""" fig, ax = plt.subplots(figsize=(10, 6)) models = [data.model_name for data in score_data_list] means = [] stds = [] for data in score_data_list: # Combine all scores from all groups all_scores = [] for group_scores in data.scores.values(): all_scores.extend(group_scores) if all_scores: scores = np.array(all_scores) scores = scores[~np.isnan(scores)] # Remove NaN values # Use appropriate error bars based on evaluation type if data.eval_type in ["infill", "lyrics", "hoot_cer", "cer"]: # For WER/CER, use standard error of the mean mean = np.mean(scores) std = np.std(scores) / np.sqrt(len(scores)) # Standard Error of Mean elif data.eval_type == "subgenre_diff": # For subgenre_diff, scores are already bootstrap samples # Use standard deviation to show the bootstrap distribution spread mean = np.mean(scores) std = np.std(scores) else: # Use bootstrap confidence interval for all other metrics mean, std = self._bootstrap_confidence_interval(scores) means.append(mean) stds.append(std) else: means.append(0) stds.append(0) # Create bars x = np.arange(len(models)) colors = [COLORS[i % len(COLORS)] for i in range(len(models))] bars = ax.bar(x, means, yerr=stds, color=colors, alpha=0.8, capsize=3) # Customize plot ax.set_xticks(x) ax.set_xticklabels(models, rotation=45, ha="right") ax.set_ylabel(config.ylabel) ax.set_title(config.title) ax.grid(True, axis="y", linestyle="--", alpha=0.3) if config.ylim: ax.set_ylim(config.ylim) plt.tight_layout() return fig, ax def _bootstrap_confidence_interval(self, data, n_bootstrap=1000, confidence=0.95): """Calculate bootstrap confidence interval for the mean.""" if len(data) == 0: return 0, 0 bootstrap_means = [] for _ in range(n_bootstrap): sample = np.random.choice(data, size=len(data), replace=True) bootstrap_means.append(np.mean(sample)) bootstrap_means = np.array(bootstrap_means) mean = np.mean(data) # Calculate confidence interval alpha = 1 - confidence lower_percentile = (alpha / 2) * 100 upper_percentile = (1 - alpha / 2) * 100 ci_lower = np.percentile(bootstrap_means, lower_percentile) ci_upper = np.percentile(bootstrap_means, upper_percentile) # Return mean and symmetric error bar (average of upper and lower deviations) error = (ci_upper - ci_lower) / 2 return mean, error def _confidence_interval_95(self, data): """Calculate 95% confidence interval for WER using t-distribution.""" if len(data) == 0: return 0, 0 from scipy import stats mean = np.mean(data) sem = np.std(data) / np.sqrt(len(data)) # Standard Error of Mean # Use t-distribution for small samples, normal for large samples if len(data) < 30: # t-distribution for small samples confidence_interval = stats.t.interval(0.95, len(data) - 1, loc=mean, scale=sem) else: # Normal distribution for large samples confidence_interval = stats.norm.interval(0.95, loc=mean, scale=sem) # Return mean and error bar (distance from mean to confidence bound) error = confidence_interval[1] - mean # Distance from mean to upper bound return mean, error def main(): parser = argparse.ArgumentParser(description="Plot evaluation scores from score files") parser.add_argument( "--input_dir", type=str, required=True, nargs="+", help="Input directories containing score files", ) parser.add_argument("--baseline_config", type=str, help="Path to baseline configuration JSON file") parser.add_argument( "--output_pdf", type=str, default="all_eval_plots.pdf", help="Output PDF filename" ) args = parser.parse_args() # Initialize components loader = ScoreLoader(args.baseline_config) plotter = PlotGenerator() # Discover and load score files print("Discovering score files...") score_files = loader.discover_score_files(args.input_dir) print(f"Found {len(score_files)} score files") # Load all score data print("Loading score data...") all_score_data = [] for filepath in score_files: score_data = loader.load_score_file(filepath) if score_data: all_score_data.append(score_data) print(f"Loaded: {score_data.model_name} ({score_data.eval_type})") # Group by evaluation type eval_groups = defaultdict(list) for score_data in all_score_data: # Skip score data with no actual scores if score_data.scores and any(score_data.scores.values()): eval_groups[score_data.eval_type].append(score_data) else: print(f"Skipping {score_data.model_name} ({score_data.eval_type}) - no scores found") # Load and merge baselines BEFORE combining subgenres (only if baseline config provided) if args.baseline_config: print("Loading baselines...") # Load baselines for the raw evaluation types (before combination) for eval_type in list(eval_groups.keys()): baselines = loader.load_baseline_scores(eval_type) for baseline_data in baselines.values(): eval_groups[eval_type].append(baseline_data) print(f"Added baseline: {baseline_data.model_name} ({eval_type})") # Also try to load baselines for common evaluation types that might not be in input common_eval_types = [ "diversity", "inst_drift", "intro", "hoot_cer", "duration", "subgenre_within", "subgenre_between", "cover", "artist_vox", "artist_self", "artist_hard", "infill", "lyrics", "cer", ] for eval_type in common_eval_types: if eval_type not in eval_groups: baselines = loader.load_baseline_scores(eval_type) if baselines: eval_groups[eval_type] = list(baselines.values()) for baseline_data in baselines.values(): print(f"Added baseline: {baseline_data.model_name} ({eval_type})") # Combine subgenre within and between into difference scores (AFTER loading baselines) eval_groups = loader.combine_subgenre_scores(eval_groups) # Generate plots print("Generating plots...") all_plots = [] for eval_type, score_list in eval_groups.items(): if not score_list: continue print(f"Creating plot for {eval_type} ({len(score_list)} models)") try: fig, ax = plotter.create_plot(score_list, eval_type) all_plots.append((fig, ax)) except Exception as e: print(f"Error creating plot for {eval_type}: {e}") # Save plots to PDF print(f"Saving plots to {args.output_pdf}...") with PdfPages(args.output_pdf) as pdf: for fig, ax in all_plots: pdf.savefig(fig, bbox_inches="tight") plt.close(fig) print(f"Saved {len(all_plots)} plots to {args.output_pdf}") if __name__ == "__main__": main()