#!/usr/bin/env python3 """ Categorize top 1000 keywords into detailed aspects for voice description analysis. """ import json from pathlib import Path from collections import defaultdict from typing import Dict, List, Tuple class KeywordCategorizer: """Categorize voice description keywords into detailed aspects.""" def __init__(self): # Define detailed category mappings self.categories = { "GENDER_AND_AGE": { "gender_primary": ["male", "female", "androgynous"], "age_primary": ["child", "teen", "young", "adult", "elderly", "mature", "youthful"], "age_specific": ["young adult", "middle-aged", "teenage", "adolescent", "preteen"], "gender_voice_type": [ "tenor", "baritone", "bass", "soprano", "alto", "mezzo-soprano", "contralto", ], }, "ACCENT_AND_ORIGIN": { "american": ["american accent", "american", "us accent", "usa"], "british": ["british accent", "british", "uk accent", "english accent"], "european": [ "german accent", "french accent", "italian accent", "spanish accent", "russian accent", "slavic accent", "eastern european", "scandinavian", ], "asian": [ "chinese accent", "japanese accent", "korean accent", "indian accent", "asian", "mandarin accent", "cantonese", ], "other_accents": [ "australian accent", "irish accent", "scottish accent", "southern accent", "new york accent", "midwest", "canadian", ], "cultural": ["latin", "african", "middle eastern", "caribbean", "jamaican"], }, "VOCAL_QUALITY": { "clarity": ["clear", "clean", "crisp", "articulate", "precise", "distinct"], "texture": [ "smooth", "rough", "raspy", "gravelly", "husky", "breathy", "airy", "velvety", "silky", "coarse", "gritty", ], "tone": [ "warm", "bright", "dark", "rich", "full", "thin", "light", "heavy", "mellow", "sharp", "soft", "hard", ], "power": [ "powerful", "strong", "gentle", "delicate", "forceful", "subtle", "bold", "quiet", "loud", "intense", ], "resonance": ["resonant", "vibrant", "hollow", "nasal", "throaty", "chesty"], }, "EMOTIONAL_EXPRESSION": { "positive_emotions": [ "happy", "joyful", "cheerful", "uplifting", "optimistic", "playful", "energetic", "enthusiastic", "excited", "fun", ], "negative_emotions": [ "sad", "melancholic", "angry", "aggressive", "dark", "moody", "somber", "mournful", "depressed", "anxious", ], "intense_emotions": [ "passionate", "emotional", "intense", "dramatic", "powerful", "fierce", "fiery", "explosive", ], "subtle_emotions": [ "intimate", "tender", "gentle", "vulnerable", "sensitive", "nostalgic", "wistful", "contemplative", "reflective", "introspective", ], "expressive_quality": [ "expressive", "emotive", "heartfelt", "soulful", "sincere", "genuine", "authentic", "raw", "honest", ], }, "MUSICAL_STYLE": { "genres": [ "pop", "rock", "jazz", "blues", "soul", "r&b", "hip-hop", "rap", "country", "folk", "metal", "punk", "electronic", "dance", "edm", "classical", "opera", "gospel", "reggae", "indie", "alternative", ], "vocal_style": [ "melodic", "harmonic", "rhythmic", "percussive", "spoken", "sung", "rapped", "shouted", "whispered", "crooned", "belted", ], "performance": [ "live", "studio", "acoustic", "unplugged", "orchestral", "choral", "solo", "duet", "harmony", "backing", ], }, "TECHNICAL_EFFECTS": { "processing": [ "reverb", "echo", "delay", "distortion", "compressed", "filtered", "modulated", "pitched", "auto-tuned", "vocoded", "doubled", "layered", ], "mixing": [ "dry", "wet", "processed", "unprocessed", "mixed", "stereo", "mono", "panned", "centered", "wide", ], "quality_descriptors": [ "lo-fi", "hi-fi", "vintage", "modern", "retro", "classic", "contemporary", "old-school", "futuristic", ], }, "PERFORMANCE_CHARACTERISTICS": { "delivery": [ "confident", "shy", "bold", "tentative", "assured", "uncertain", "steady", "shaky", "controlled", "wild", ], "energy": [ "dynamic", "static", "energetic", "laid-back", "relaxed", "tense", "calm", "excited", "manic", "subdued", ], "authenticity": [ "natural", "theatrical", "dramatic", "conversational", "formal", "casual", "professional", "amateur", ], "technique": [ "vibrato", "falsetto", "belting", "growling", "screaming", "whispering", "humming", "scat", "yodeling", "melisma", ], }, "DESCRIPTIVE_QUALITIES": { "overall_impression": [ "beautiful", "ugly", "pleasant", "unpleasant", "unique", "distinctive", "generic", "memorable", "forgettable", ], "character": [ "charismatic", "charming", "mysterious", "haunting", "ethereal", "earthy", "angelic", "demonic", "robotic", "human", ], "cultural_style": [ "urban", "rural", "street", "sophisticated", "primitive", "modern", "traditional", "contemporary", "ancient", ], }, "VOCAL_RANGE": { "pitch": [ "high", "low", "mid", "deep", "high-pitched", "low-pitched", "mid-range", "baritone", "bass", "tenor", "soprano", "alto", ], "range_descriptors": ["wide range", "narrow range", "full range", "limited range"], }, "LANGUAGE_AND_DICTION": { "language": [ "english", "spanish", "french", "german", "italian", "chinese", "japanese", "korean", "russian", "portuguese", "arabic", ], "diction": ["clear diction", "mumbled", "slurred", "articulate", "enunciated"], "linguistic_features": ["accented", "non-native", "fluent", "broken", "pidgin"], }, } def categorize_keyword(self, keyword: str) -> List[Tuple[str, str]]: """ Categorize a single keyword into its aspects. Returns: List of (main_category, subcategory) tuples """ keyword_lower = keyword.lower().strip() matches = [] for main_cat, subcats in self.categories.items(): for subcat, keywords_list in subcats.items(): for cat_keyword in keywords_list: if cat_keyword in keyword_lower or keyword_lower in cat_keyword: matches.append((main_cat, subcat)) break # If no matches, try partial matching if not matches: # Check for partial word matches words = keyword_lower.split() for word in words: if len(word) > 3: # Only check words longer than 3 characters for main_cat, subcats in self.categories.items(): for subcat, keywords_list in subcats.items(): for cat_keyword in keywords_list: if word in cat_keyword or cat_keyword in word: matches.append((main_cat, subcat)) break if matches: break if matches: break return matches if matches else [("UNCATEGORIZED", "other")] def analyze_top_keywords(self, input_file: str, output_dir: str): """Analyze and categorize top 1000 keywords.""" output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) # Load top 1000 keywords with open(input_file, "r") as f: data = json.load(f) # Get the top 1000 keywords if "top_1000_keywords" in data: keywords = data["top_1000_keywords"] elif "keyword_frequencies" in data: # If using all_keywords file, get top 1000 all_keywords = data["keyword_frequencies"] keywords = dict(sorted(all_keywords.items(), key=lambda x: x[1], reverse=True)[:1000]) else: raise ValueError("Could not find keywords in input file") # Categorize each keyword categorized = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) category_totals = defaultdict(int) uncategorized_keywords = [] for keyword, count in keywords.items(): categories = self.categorize_keyword(keyword) if categories[0][0] == "UNCATEGORIZED": uncategorized_keywords.append((keyword, count)) else: for main_cat, subcat in categories: categorized[main_cat][subcat][keyword] = count category_totals[main_cat] += count # Create detailed report report = { "summary": { "total_keywords_analyzed": len(keywords), "total_occurrences": sum(keywords.values()), "categorized_keywords": len(keywords) - len(uncategorized_keywords), "uncategorized_keywords": len(uncategorized_keywords), "main_categories": len(categorized), }, "category_distribution": {}, "detailed_categories": {}, "uncategorized": {}, "top_keywords_by_category": {}, } # Process each category for main_cat in sorted(categorized.keys()): subcats = categorized[main_cat] # Calculate statistics total_keywords = sum(len(keywords) for keywords in subcats.values()) total_occurrences = category_totals[main_cat] report["category_distribution"][main_cat] = { "total_keywords": total_keywords, "total_occurrences": total_occurrences, "percentage_of_total": round(100 * total_occurrences / sum(keywords.values()), 2), "subcategories": len(subcats), } # Detailed breakdown report["detailed_categories"][main_cat] = {} report["top_keywords_by_category"][main_cat] = {} for subcat in sorted(subcats.keys()): keywords_dict = subcats[subcat] sorted_keywords = sorted(keywords_dict.items(), key=lambda x: x[1], reverse=True) report["detailed_categories"][main_cat][subcat] = { "keyword_count": len(keywords_dict), "total_occurrences": sum(keywords_dict.values()), "keywords": dict(sorted_keywords), } # Top 5 keywords per subcategory report["top_keywords_by_category"][main_cat][subcat] = dict(sorted_keywords[:5]) # Add uncategorized keywords report["uncategorized"] = { "count": len(uncategorized_keywords), "keywords": dict(sorted(uncategorized_keywords, key=lambda x: x[1], reverse=True)[:100]), } # Save main report report_file = output_path / "top_1000_detailed_categories.json" with open(report_file, "w") as f: json.dump(report, f, indent=2) print(f"Saved detailed categorization to {report_file}") # Create summary visualization self.create_summary_report(report, output_path) return report def create_summary_report(self, report: Dict, output_path: Path): """Create a human-readable summary report.""" summary_file = output_path / "category_summary.txt" with open(summary_file, "w") as f: f.write("=" * 80 + "\n") f.write("VOICE KEYWORD CATEGORIZATION SUMMARY\n") f.write("Top 1000 Keywords Analysis\n") f.write("=" * 80 + "\n\n") # Overall statistics f.write("OVERALL STATISTICS:\n") f.write("-" * 40 + "\n") for key, value in report["summary"].items(): f.write(f"{key.replace('_', ' ').title()}: {value:,}\n") f.write("\n") # Category distribution f.write("CATEGORY DISTRIBUTION:\n") f.write("-" * 40 + "\n") sorted_cats = sorted( report["category_distribution"].items(), key=lambda x: x[1]["total_occurrences"], reverse=True, ) for cat, stats in sorted_cats: f.write(f"\n{cat}:\n") f.write(f" Keywords: {stats['total_keywords']:,}\n") f.write(f" Occurrences: {stats['total_occurrences']:,}\n") f.write(f" Percentage: {stats['percentage_of_total']:.1f}%\n") f.write(f" Subcategories: {stats['subcategories']}\n") # Show top keywords for this category if cat in report["top_keywords_by_category"]: f.write(" Top keywords by subcategory:\n") for subcat, keywords in report["top_keywords_by_category"][cat].items(): if keywords: top_keyword = list(keywords.items())[0] f.write(f" {subcat}: {top_keyword[0]} ({top_keyword[1]:,})\n") # Uncategorized keywords f.write("\n" + "=" * 80 + "\n") f.write(f"UNCATEGORIZED KEYWORDS: {report['uncategorized']['count']}\n") f.write("-" * 40 + "\n") if report["uncategorized"]["keywords"]: f.write("Top uncategorized keywords:\n") for keyword, count in list(report["uncategorized"]["keywords"].items())[:20]: f.write(f" {keyword}: {count:,}\n") print(f"Saved summary to {summary_file}") def main(): """Main execution.""" categorizer = KeywordCategorizer() # Analyze top 1000 keywords report = categorizer.analyze_top_keywords( "/home/vibert/tmp/voice_keywords_analysis/top_1000_keywords.json", "/home/vibert/tmp/voice_keywords_analysis", ) print("\n" + "=" * 60) print("Categorization Complete!") print("=" * 60) print(f"Analyzed: {report['summary']['total_keywords_analyzed']} keywords") print(f"Categorized: {report['summary']['categorized_keywords']} keywords") print(f"Uncategorized: {report['summary']['uncategorized_keywords']} keywords") print(f"Main categories: {report['summary']['main_categories']}") print("\nFiles created:") print(" - top_1000_detailed_categories.json") print(" - category_summary.txt") if __name__ == "__main__": main()