{ "cells": [ { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "# Suno User Clustering v0.7 - Full User Base Analysis (Free + Paid)\n", "\n", "## ๐Ÿ“‹ Executive Summary\n", "\n", "This notebook implements comprehensive user clustering for Suno, analyzing **ALL users including free tier and non-creators**. We identify key user segments based on multi-dimensional engagement patterns across the entire user base.\n", "\n", "### ๐Ÿ”ง Key Improvements in v0.7:\n", "- โœ… **Full User Base**: Includes FREE users and non-creators (not just paid creators)\n", "- โœ… **Creator Flag**: Distinguishes creators from consumers/lurkers\n", "- โœ… **Free Tier Analysis**: Captures free-to-paid conversion opportunities\n", "- โœ… **Non-Creator Insights**: Understands passive users and potential converts\n", "\n", "### Expected User Segments:\n", "- ๐Ÿ†“ **Free Users**: Non-paying users, limited usage, conversion targets\n", "- ๐Ÿ‘€ **Lurkers/Consumers**: Users who consume but don't create content\n", "- ๐ŸŽฏ **Pro Power Users**: Advanced model users (v4p5), high-quality content creators\n", "- ๐ŸŽช **Casual Creators**: Experimenting with AI music, mixed free/paid\n", "- ๐Ÿค– **Automation Users**: High-volume API generation, minimal consumption\n", "- ๐Ÿš€ **Super Creators**: Prolific content producers, platform champions\n", "- ๐Ÿ“ข **Music Influencers**: High sharing activity, public content, community builders\n", "\n", "---\n" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ“Š Section 1: Setup and Data Loading\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# Setup autoload for development\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "๐Ÿš€ SUNO USER CLUSTERING v0.6 (Standardized API Edition)\n", "============================================================\n", "๐Ÿ“… Analysis Date: 2025-06-17 | Target clusters: [4, 5]\n" ] } ], "source": [ "# Standard library imports\n", "import pandas as pd\n", "import numpy as np\n", "from pathlib import Path\n", "import gc\n", "from datetime import datetime, timedelta\n", "import warnings\n", "from typing import Dict, List, Optional, Tuple\n", "import os\n", "\n", "# Sklearn imports\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.cluster import MiniBatchKMeans\n", "from sklearn.metrics import silhouette_score\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import f1_score\n", "\n", "# Visualization imports\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "# Our custom modules\n", "from user_selection import UserSelector\n", "from reaction_features import ReactionFeatureExtractor\n", "from content_features import ContentFeatureExtractor\n", "from bot_features import BotFeatureExtractor\n", "from engagement_features import EngagementFeatureCreator\n", "from clustering import UserClusterer\n", "from feature_utils import safe_merge_features, print_feature_summary\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "# Configuration\n", "TARGET_CLUSTERS = range(4, 6) # 4-6 clusters based on expected archetypes\n", "MIN_SILHOUETTE = 0.10 # Adjusted for high-dimensional data\n", "SAMPLE_SIZE = 5000 # Reduced to prevent kernel crashes with 290k+ users\n", "ANALYSIS_DATE = pd.to_datetime(\"2025-06-17\")\n", "\n", "# Data paths\n", "data_dir = Path(\"/home/tony/Data/Usercluster/sample_20250617/\")\n", "interesting_clips_path = Path(\n", " \"/home/tony/Data/Usercluster/sample_20250617/total_clip.pkl\"\n", ")\n", "\n", "print(\"๐Ÿš€ SUNO USER CLUSTERING v0.7 (Full User Base Edition)\")\n", "print(\"=\" * 60)\n", "print(\n", " f\"๐Ÿ“… Analysis Date: {ANALYSIS_DATE.date()} | Target clusters: {list(TARGET_CLUSTERS)}\"\n", ")\n", "print(\"๐Ÿ†• Now including FREE users and non-creators!\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ“ Loading datasets...\n", " Loading total_clip.pkl...\n", " Loading discord_info.pkl...\n", " Loading reaction.pkl...\n", " Loading bots_action.pkl...\n", "\n", " โœ… Successfully loaded all files:\n", " โ€ข Users: 674,490\n", " โ€ข Reactions: 50,243,073\n", " โ€ข Bot actions: 41,984,051\n", " โ€ข Clips: 41,901,309\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Load datasets\n", "print(\"\\n๐Ÿ“ Loading datasets...\")\n", "\n", "try:\n", " print(\" Loading total_clip.pkl...\")\n", " total_clip_df = pd.read_pickle(interesting_clips_path)\n", "\n", " print(\" Loading discord_info.pkl...\")\n", " discord_info_df = pd.read_pickle(data_dir / \"discord_info.pkl\")\n", "\n", " print(\" Loading reaction.pkl...\")\n", " reaction_df = pd.read_pickle(data_dir / \"reaction.pkl\")\n", "\n", " print(\" Loading bots_action.pkl...\")\n", " bots_action_df = pd.read_pickle(data_dir / \"bots_action.pkl\")\n", "\n", " print(f\"\\n โœ… Successfully loaded all files:\")\n", " print(f\" โ€ข Users: {len(discord_info_df):,}\")\n", " print(f\" โ€ข Reactions: {len(reaction_df):,}\")\n", " print(f\" โ€ข Bot actions: {len(bots_action_df):,}\")\n", " print(f\" โ€ข Clips: {len(total_clip_df):,}\")\n", "\n", "except Exception as e:\n", " print(f\"\\n โŒ Error loading data: {type(e).__name__}: {e}\")\n", " raise\n", "\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐ŸŽฏ Section 2: User Selection (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "๐ŸŽฏ USER SUBSET SELECTION\n", "==================================================\n", "๐ŸŽฏ USER SUBSET SELECTION - CREATORS ONLY\n", "==================================================\n", "๐Ÿ“Š Content creators selected: 290,046 users\n", "๐Ÿ“Š Total unique creators in clips dataset: 2,054,644\n", "๐Ÿ“Š Valid creators (exist in user database): 290,046\n", "๐Ÿ“Š This represents 43.0% of all users in the system\n", "โœ… Focus: Analyzing actual content creators from interesting clips dataset\n", "๐Ÿ’ณ SUBSCRIPTION FEATURES\n", "==================================================\n", "๐Ÿ“Š Distribution: Free=0 | Past Due=1,918 | Active=288,128\n", "โœ… Shape: (290046, 2)\n", "\n", "โœ… Initial features shape: (290046, 2)\n", "\n", "๐Ÿ“Š User Statistics Summary:\n", " โ€ข Total creators: 290,046\n", " โ€ข Active subscribers: 99.3%\n", " โ€ข Paying users: 100.0%\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: UserSelector returns DataFrame directly\n", "print(\"๐ŸŽฏ USER SUBSET SELECTION\")\n", "print(\"=\" * 50)\n", "\n", "user_selector = UserSelector()\n", "\n", "# Create initial features - NOW INCLUDING ALL USERS (free + paid)\n", "features_df = user_selector.create_initial_features(\n", " total_clip_df, discord_info_df, include_all_users=True, verbose=True\n", ")\n", "\n", "# Verify we have a DataFrame\n", "assert isinstance(features_df, pd.DataFrame), \"features_df should be a DataFrame\"\n", "print(f\"\\nโœ… Initial features shape: {features_df.shape}\")\n", "\n", "# Get user statistics\n", "user_stats = user_selector.get_user_statistics(features_df)\n", "print(f\"\\n๐Ÿ“Š User Statistics Summary:\")\n", "print(f\" โ€ข Total users: {user_stats['total_users']:,}\")\n", "print(\n", " f\" โ€ข Creators: {(features_df['is_creator'] == 1).sum():,} ({(features_df['is_creator'] == 1).mean()*100:.1f}%)\"\n", ")\n", "print(\n", " f\" โ€ข Non-creators: {(features_df['is_creator'] == 0).sum():,} ({(features_df['is_creator'] == 0).mean()*100:.1f}%)\"\n", ")\n", "print(f\" โ€ข Active subscribers: {user_stats['active_subscriber_pct']:.1f}%\")\n", "print(f\" โ€ข Paying users: {user_stats['paying_user_pct']:.1f}%\")\n", "print(f\" โ€ข Free users: {100 - user_stats['paying_user_pct']:.1f}%\")\n", "\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ‘ Section 3: Reaction Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ‘ REACTION FEATURES\n", "==================================================\n", " Processing reactions for filtered users...\n", " Available columns: ['clip_id', 'flagged', 'flagged_reason', 'is_pro_user', 'play_count', 'reaction_type', 'skip_count', 'updated_at', 'user_id']\n", "\n", "โœ… Extracted reaction features: (287688, 18)\n", " โ€ข Users with reactions: 287,688\n", "\n", "โœ… Features after reaction merge: (290046, 14)\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: ReactionFeatureExtractor returns tuple (DataFrame, dict)\n", "print(\"\\n๐Ÿ‘ REACTION FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "reaction_extractor = ReactionFeatureExtractor()\n", "\n", "# Extract features - RETURNS TUPLE!\n", "reaction_features_df, reaction_summary = reaction_extractor.extract_features(\n", " reaction_df=reaction_df, user_ids=features_df[\"user_id\"], verbose=True\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " reaction_features_df, pd.DataFrame\n", "), \"reaction_features_df should be a DataFrame\"\n", "assert isinstance(reaction_summary, dict), \"reaction_summary should be a dict\"\n", "\n", "print(f\"\\nโœ… Extracted reaction features: {reaction_features_df.shape}\")\n", "print(f\" โ€ข Users with reactions: {reaction_summary.get('users_with_reactions', 0):,}\")\n", "\n", "# Merge using extractor's merge method\n", "features_df = reaction_extractor.merge_features(\n", " features_df=features_df, reaction_stats=reaction_features_df, check_existing=True\n", ")\n", "\n", "print(f\"\\nโœ… Features after reaction merge: {features_df.shape}\")\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐ŸŽต Section 4: Content Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐ŸŽต CONTENT CREATION FEATURES\n", "==================================================\n", " Processing clips from filtered users...\n", " Available columns: ['allow_comments', 'batch_index', 'clip_type', 'continued_parent', 'created_at', 'created_session_id', 'daily_theme_id', 'date', 'discord_message_id', 'dislike_count']...\n", "\n", "โœ… Extracted content features: (290046, 34)\n", " โ€ข Sample features: ['user_id', 'total_clips_created', 'days_creating', 'is_recent_creator', 'clip_creation_rate']\n", "\n", "๐Ÿ“Š Content Summary:\n", " โ€ข Total creators: 290,046\n", " โ€ข Active creators: 290,046\n", " โ€ข Recent creators: 290,046\n", "\n", "โœ… Features after content merge: (290046, 47)\n" ] }, { "data": { "text/plain": [ "55" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: ContentFeatureExtractor returns tuple (DataFrame, dict)\n", "print(\"\\n๐ŸŽต CONTENT CREATION FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "content_extractor = ContentFeatureExtractor(analysis_date=ANALYSIS_DATE)\n", "\n", "# Extract features - RETURNS TUPLE!\n", "content_features_df, content_summary = content_extractor.extract_features(\n", " total_clip_df=total_clip_df, user_ids=features_df[\"user_id\"], verbose=True\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " content_features_df, pd.DataFrame\n", "), \"content_features_df should be a DataFrame\"\n", "assert isinstance(content_summary, dict), \"content_summary should be a dict\"\n", "\n", "print(f\"\\nโœ… Extracted content features: {content_features_df.shape}\")\n", "print(f\" โ€ข Sample features: {list(content_features_df.columns[:5])}\")\n", "\n", "# Print summary\n", "if content_summary:\n", " print(f\"\\n๐Ÿ“Š Content Summary:\")\n", " print(f\" โ€ข Total creators: {content_summary.get('total_creators', 0):,}\")\n", " print(f\" โ€ข Active creators: {content_summary.get('active_creators', 0):,}\")\n", " print(f\" โ€ข Recent creators: {content_summary.get('recent_creators', 0):,}\")\n", "\n", "# Merge directly (content doesn't have merge_features method in use)\n", "features_df = features_df.merge(content_features_df, on=\"user_id\", how=\"left\")\n", "\n", "print(f\"\\nโœ… Features after content merge: {features_df.shape}\")\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿค– Section 5: Bot Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿค– BOT INTERACTION FEATURES\n", "==================================================\n", " Checking clip ID mapping...\n", " Available bot action columns: ['clip_id', 'created_at', 'download_audio_count', 'download_audio_wav_count', 'download_video_count', 'share_count', 'updated_at', 'user_id']...\n", "๐Ÿ“Š Found: 23,892,041 bot actions from 289,621 users\n", "๐Ÿ“Š Downloads: 177,941 users | Avg: 16.0/user\n", "๐Ÿ“Š Shares: 47,366 users | Avg: 4.3/user\n", "\n", "โœ… Extracted bot features: (289621, 23)\n", " โ€ข Sample features: ['user_id', 'total_bot_actions', 'download_audio_count_sum', 'download_audio_count_mean', 'download_video_count_sum']\n", "\n", "๐Ÿ“Š Bot Summary:\n", " โ€ข Bot users: 289,621\n", " โ€ข Power users (API tier 3+): 119,740\n", "\n", "โœ… Features after bot merge: (290046, 59)\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: BotFeatureExtractor returns tuple (DataFrame, dict)\n", "print(\"\\n๐Ÿค– BOT INTERACTION FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "bot_extractor = BotFeatureExtractor()\n", "\n", "# Extract features - RETURNS TUPLE!\n", "bot_features_df, bot_summary = bot_extractor.extract_features(\n", " bots_action_df=bots_action_df,\n", " total_clip_df=total_clip_df,\n", " user_ids=features_df[\"user_id\"],\n", " features_df=features_df,\n", " verbose=True,\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " bot_features_df, pd.DataFrame\n", "), \"bot_features_df should be a DataFrame\"\n", "assert isinstance(bot_summary, dict), \"bot_summary should be a dict\"\n", "\n", "print(f\"\\nโœ… Extracted bot features: {bot_features_df.shape}\")\n", "print(f\" โ€ข Sample features: {list(bot_features_df.columns[:5])}\")\n", "\n", "# Print summary\n", "if bot_summary:\n", " print(f\"\\n๐Ÿ“Š Bot Summary:\")\n", " print(f\" โ€ข Bot users: {bot_summary.get('bot_users', 0):,}\")\n", " if \"power_users\" in bot_summary:\n", " print(f\" โ€ข Power users (API tier 3+): {bot_summary['power_users']:,}\")\n", "\n", "# Merge using extractor's merge method\n", "features_df = bot_extractor.merge_features(\n", " features_df=features_df, bot_features=bot_features_df\n", ")\n", "\n", "print(f\"\\nโœ… Features after bot merge: {features_df.shape}\")\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ“Š Section 6: Engagement Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ“Š ENGAGEMENT SUMMARY FEATURES\n", "==================================================\n", " Creating engagement summary features...\n", " Key columns check:\n", " โ€ข play_frequency: True\n", " โ€ข share_count: True\n", " โ€ข total_downloads: True\n", " โ€ข model_diversity: True\n", "\n", "โœ… Features after engagement creation: (290046, 75)\n", "\n", "๐Ÿ“Š Engagement Summary:\n", " โ€ข Total features: 74\n", " โ€ข Avg activity diversity: 5.33\n", " โ€ข Users with 5+ activities: 242,491\n", "\n", " User Segments:\n", " โ€ข super_creator: 118,318 (40.8%)\n", " โ€ข casual_experimenter: 89,760 (30.9%)\n", " โ€ข dormant_user: 40,147 (13.8%)\n", " โ€ข casual_creator: 28,633 (9.9%)\n", " โ€ข pro_power_user: 10,479 (3.6%)\n", " โ€ข regular_active_user: 2,709 (0.9%)\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: EngagementFeatureCreator returns tuple (DataFrame, dict)\n", "# IMPORTANT: Returns UPDATED DataFrame, not just new features!\n", "print(\"\\n๐Ÿ“Š ENGAGEMENT SUMMARY FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "engagement_extractor = EngagementFeatureCreator()\n", "\n", "# Create features - RETURNS TUPLE with UPDATED DataFrame!\n", "features_df_updated, engagement_summary = (\n", " engagement_extractor.create_engagement_features(\n", " features_df=features_df, verbose=True\n", " )\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " features_df_updated, pd.DataFrame\n", "), \"features_df_updated should be a DataFrame\"\n", "assert isinstance(engagement_summary, dict), \"engagement_summary should be a dict\"\n", "\n", "# CRITICAL: Update features_df with the returned DataFrame\n", "features_df = features_df_updated\n", "\n", "print(f\"\\nโœ… Features after engagement creation: {features_df.shape}\")\n", "\n", "# Print summary\n", "if engagement_summary:\n", " print(f\"\\n๐Ÿ“Š Engagement Summary:\")\n", " print(f\" โ€ข Total features: {engagement_summary.get('total_features', 0)}\")\n", " print(\n", " f\" โ€ข Avg activity diversity: {engagement_summary.get('avg_activity_diversity', 0):.2f}\"\n", " )\n", " print(\n", " f\" โ€ข Users with 5+ activities: {engagement_summary.get('users_with_5plus_activities', 0):,}\"\n", " )\n", "\n", " if \"user_segments\" in engagement_summary:\n", " print(f\"\\n User Segments:\")\n", " for segment, data in engagement_summary[\"user_segments\"].items():\n", " print(f\" โ€ข {segment}: {data['count']:,} ({data['percentage']:.1f}%)\")\n", "\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ” Section 7: Feature Availability Check\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ” FEATURE AVAILABILITY CHECK\n", "==================================================\n", "โœ… features_df is a valid DataFrame with shape: (290046, 75)\n", "๐Ÿ“Š Total features available: 74\n", "\n", "๐Ÿ“Š Feature Availability by Category:\n", "\n", "User Features:\n", " โœ… Available: 2/2\n", "\n", "Reaction Features:\n", " โœ… Available: 5/5\n", "\n", "Content Features:\n", " โœ… Available: 6/6\n", "\n", "Bot Features:\n", " โœ… Available: 5/5\n", "\n", "Engagement Features:\n", " โœ… Available: 5/5\n", "\n", "๐Ÿ“Š Overall Coverage: 23/23 (100.0%)\n", "๐Ÿ“Š Total Columns: 75\n" ] } ], "source": [ "# Safe feature availability check\n", "print(\"\\n๐Ÿ” FEATURE AVAILABILITY CHECK\")\n", "print(\"=\" * 50)\n", "\n", "# Ensure we have a DataFrame before checking columns\n", "if not isinstance(features_df, pd.DataFrame):\n", " print(\"โŒ ERROR: features_df is not a DataFrame!\")\n", " print(f\" Type: {type(features_df)}\")\n", " raise TypeError(\"features_df should be a DataFrame at this point\")\n", "\n", "# Now safe to check columns\n", "print(f\"โœ… features_df is a valid DataFrame with shape: {features_df.shape}\")\n", "print(f\"๐Ÿ“Š Total features available: {len(features_df.columns) - 1}\") # Exclude user_id\n", "\n", "# Define expected features by category\n", "expected_features = {\n", " \"User\": [\"user_id\", \"subscription_tier\", \"is_creator\"],\n", " \"Reaction\": [\n", " \"reaction_like_ratio\",\n", " \"reaction_dislike_ratio\",\n", " \"reaction_frequency\",\n", " \"creator_feedback_ratio\",\n", " \"community_interaction_score\",\n", " ],\n", " \"Content\": [\n", " \"total_clips_created\",\n", " \"clip_creation_rate\",\n", " \"is_recent_creator\",\n", " \"model_diversity\",\n", " \"task_diversity\",\n", " \"source_diversity\",\n", " ],\n", " \"Bot\": [\n", " \"total_bot_actions\",\n", " \"bot_action_rate\",\n", " \"api_usage_tier\",\n", " \"total_downloads\",\n", " \"share_count\",\n", " ],\n", " \"Engagement\": [\n", " \"engagement_score\",\n", " \"user_segment_encoded\",\n", " \"activity_diversity\",\n", " \"content_diversity_score\",\n", " \"viral_potential\",\n", " ],\n", "}\n", "\n", "# Check availability\n", "available_features = set(features_df.columns)\n", "total_expected = 0\n", "total_available = 0\n", "\n", "print(\"\\n๐Ÿ“Š Feature Availability by Category:\\n\")\n", "for category, features in expected_features.items():\n", " available = [f for f in features if f in available_features]\n", " missing = [f for f in features if f not in available_features]\n", " total_expected += len(features)\n", " total_available += len(available)\n", "\n", " print(f\"{category} Features:\")\n", " print(f\" โœ… Available: {len(available)}/{len(features)}\")\n", " if missing:\n", " print(\n", " f\" โŒ Missing: {', '.join(missing[:3])}{'...' if len(missing) > 3 else ''}\"\n", " )\n", " print()\n", "\n", "print(\n", " f\"๐Ÿ“Š Overall Coverage: {total_available}/{total_expected} ({total_available/total_expected*100:.1f}%)\"\n", ")\n", "print(f\"๐Ÿ“Š Total Columns: {len(features_df.columns)}\")" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ” Section 8: Mini-Batch K-Means Clustering\n" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "### ๐Ÿšจ Robust Clustering Settings\n", "\n", "The clustering is configured with memory-efficient settings to prevent kernel crashes:\n", "- **Reduced sample size**: 5,000 users (down from 10,000)\n", "- **Conservative batch sizes**: Automatically adjusted based on data size\n", "- **Memory limit**: 4GB maximum memory usage\n", "- **Garbage collection**: Automatic cleanup after each k-value test\n", "- **Error handling**: Graceful fallback if clustering fails\n", "\n", "These settings ensure stable clustering even with 290,000+ users.\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ” MINI-BATCH K-MEANS CLUSTERING\n", "==================================================\n", "โœ… Found 23 clustering features\n", "๐Ÿ“Š Feature availability:\n", " Total defined: 23\n", " Available: 23\n", " Missing: 0\n", " โš ๏ธ Large dataset detected (290,046 users)\n", " Using reduced sample size: 5,000\n", "\n", "๐Ÿ“Š Clustering data: 23 features | 5,000 samples\n", "๐Ÿ“Š Sample size for clustering: 5,000 users\n", "๐Ÿ“Š Features used: 23\n", "\n", "๐Ÿ” Testing k=[4, 5] with memory-efficient settings...\n", " Testing k=4... silhouette=0.146\n", " Testing k=5... silhouette=0.202\n", "\n", "๐ŸŽฏ Best: k=5, silhouette=0.202 (โœ… target: 0.1)\n", " Batch size: 1000, Sample size: 2500\n", "\n", "โœ… Optimal clustering: k=5 with silhouette=0.202\n", "\n", "๐Ÿ“Š Applying clustering to full dataset...\n", " Processing 290,046 users...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n", "OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Processed 100,000 users...\n", " Processed 200,000 users...\n", " โœ… Clustering complete for all 290,046 users\n", "\n", "๐Ÿ“Š DETAILED CLUSTER ANALYSIS:\n", "================================================================================\n", "\n", "================================================================================\n", "CLUSTER 4 - 33.0 users (0.7%)\n", "================================================================================\n", "\n", "๐Ÿ“Š Key Metrics:\n", " โ€ข Engagement Score: 15310.3\n", " โ€ข Avg Clips Created: 1584.2\n", " โ€ข Reaction Frequency: 4.09/day\n", " โ€ข Downloads: 26.8\n", " โ€ข Shares: 0.5\n", "\n", "๐Ÿค– Model Usage:\n", " โ€ข Advanced Model Usage: 12.5%\n", " โ€ข v4p5 Usage: 0.0%\n", "\n", "๐Ÿ“ข Visibility:\n", " โ€ข Public Content: 0.0%\n", "\n", "๐Ÿ’ก Interpretation: Elite engagement tier | mega content producers\n", "\n", "================================================================================\n", "CLUSTER 3 - 86.0 users (1.7%)\n", "================================================================================\n", "\n", "๐Ÿ“Š Key Metrics:\n", " โ€ข Engagement Score: 5530.0\n", " โ€ข Avg Clips Created: 546.2\n", " โ€ข Reaction Frequency: 1.79/day\n", " โ€ข Downloads: 166.8\n", " โ€ข Shares: 2.3\n", "\n", "๐Ÿค– Model Usage:\n", " โ€ข Advanced Model Usage: 2.0%\n", " โ€ข v4p5 Usage: 0.0%\n", "\n", "๐Ÿ“ข Visibility:\n", " โ€ข Public Content: 0.0%\n", "\n", "๐Ÿ’ก Interpretation: Elite engagement tier | mega content producers\n", "\n", "================================================================================\n", "CLUSTER 2 - 1,831.0 users (36.6%)\n", "================================================================================\n", "\n", "๐Ÿ“Š Key Metrics:\n", " โ€ข Engagement Score: 1479.1\n", " โ€ข Avg Clips Created: 137.0\n", " โ€ข Reaction Frequency: 0.86/day\n", " โ€ข Downloads: 14.5\n", " โ€ข Shares: 1.3\n", "\n", "๐Ÿค– Model Usage:\n", " โ€ข Advanced Model Usage: 6.5%\n", " โ€ข v4p5 Usage: 0.0%\n", "\n", "๐Ÿ“ข Visibility:\n", " โ€ข Public Content: 0.0%\n", "\n", "๐Ÿ’ก Interpretation: Elite engagement tier | prolific creators\n", "\n", "================================================================================\n", "CLUSTER 0 - 856.0 users (17.1%)\n", "================================================================================\n", "\n", "๐Ÿ“Š Key Metrics:\n", " โ€ข Engagement Score: 355.6\n", " โ€ข Avg Clips Created: 28.1\n", " โ€ข Reaction Frequency: 0.28/day\n", " โ€ข Downloads: 4.3\n", " โ€ข Shares: 0.9\n", "\n", "๐Ÿค– Model Usage:\n", " โ€ข Advanced Model Usage: 3.9%\n", " โ€ข v4p5 Usage: 0.0%\n", "\n", "๐Ÿ“ข Visibility:\n", " โ€ข Public Content: 0.0%\n", "\n", "๐Ÿ’ก Interpretation: High engagement tier | regular creators\n", "\n", "================================================================================\n", "CLUSTER 1 - 2,194.0 users (43.9%)\n", "================================================================================\n", "\n", "๐Ÿ“Š Key Metrics:\n", " โ€ข Engagement Score: 258.9\n", " โ€ข Avg Clips Created: 19.9\n", " โ€ข Reaction Frequency: 0.12/day\n", " โ€ข Downloads: 2.4\n", " โ€ข Shares: 0.2\n", "\n", "๐Ÿค– Model Usage:\n", " โ€ข Advanced Model Usage: 3.8%\n", " โ€ข v4p5 Usage: 0.0%\n", "\n", "๐Ÿ“ข Visibility:\n", " โ€ข Public Content: 0.0%\n", "\n", "๐Ÿ’ก Interpretation: High engagement tier | casual creators\n", "\n", "๐Ÿ“Š Final Cluster Distribution:\n", " โ€ข High_Regular: 49,614 users (17.1%)\n", " โ€ข High_Casual: 125,676 users (43.3%)\n", " โ€ข Elite_Prolific: 108,233 users (37.3%)\n", " โ€ข Elite_MegaCreators_Collectors: 4,573 users (1.6%)\n", " โ€ข Elite_MegaCreators_Interactive: 1,950 users (0.7%)\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Perform clustering with proper DataFrame handling\n", "print(\"\\n๐Ÿ” MINI-BATCH K-MEANS CLUSTERING\")\n", "print(\"=\" * 50)\n", "\n", "# Verify features_df is still a DataFrame\n", "assert isinstance(\n", " features_df, pd.DataFrame\n", "), \"features_df must be a DataFrame for clustering\"\n", "\n", "# Initialize clusterer with memory-efficient settings\n", "# Using reduced sample size and conservative settings to prevent kernel crashes\n", "clusterer = UserClusterer(\n", " target_clusters=TARGET_CLUSTERS,\n", " min_silhouette=MIN_SILHOUETTE,\n", " sample_size=5000, # Reduced from 10000 to prevent crashes\n", " random_state=42,\n", " max_memory_gb=4.0, # Limit memory usage\n", ")\n", "\n", "# Define clustering features (only those that exist)\n", "all_clustering_features = [\n", " # Core features\n", " \"is_creator\", # NEW: Distinguish creators from non-creators\n", " \"user_segment_encoded\",\n", " \"total_clips_created\",\n", " \"clip_creation_rate\",\n", " \"is_recent_creator\",\n", " \"total_bot_actions\",\n", " \"bot_action_rate\",\n", " \"api_usage_tier\",\n", " \"reaction_frequency\",\n", " \"subscription_tier\",\n", " \"engagement_score\",\n", " \"activity_diversity\",\n", " # Model features\n", " \"model_diversity\",\n", " \"task_diversity\",\n", " \"source_diversity\",\n", " \"advanced_model_ratio\",\n", " \"v4p5_ratio\",\n", " # Distribution features\n", " \"total_downloads\",\n", " \"share_count\",\n", " \"public_clip_ratio\",\n", " # Derived features\n", " \"content_velocity\",\n", " \"sharing_propensity\",\n", " \"viral_potential\",\n", " \"creator_consumer_ratio\",\n", " \"platform_loyalty_score\",\n", "]\n", "\n", "# Filter to available features\n", "available_clustering_features = [\n", " f for f in all_clustering_features if f in features_df.columns\n", "]\n", "print(f\"โœ… Found {len(available_clustering_features)} clustering features\")\n", "\n", "# Prepare features\n", "features_sample, final_features = clusterer.prepare_features(\n", " features_df, available_clustering_features, verbose=True\n", ")\n", "features_sample.fillna(0, inplace=True)\n", "\n", "# Find optimal clusters with error handling\n", "try:\n", " # Ensure we pass a DataFrame, not a Series\n", " X_sample = features_sample[final_features]\n", " if isinstance(X_sample, pd.Series):\n", " X_sample = X_sample.to_frame()\n", "\n", " print(f\"๐Ÿ“Š Sample size for clustering: {len(X_sample):,} users\")\n", " print(f\"๐Ÿ“Š Features used: {len(final_features)}\")\n", "\n", " best_k, best_silhouette, best_model = clusterer.find_optimal_clusters(\n", " X_sample, verbose=True\n", " )\n", "except MemoryError:\n", " print(\"\\nโŒ Memory error during clustering! Reducing sample size...\")\n", " # Try with even smaller sample\n", " smaller_sample = features_sample.sample(n=2000, random_state=42)\n", " X_sample = smaller_sample[final_features]\n", " best_k, best_silhouette, best_model = clusterer.find_optimal_clusters(\n", " X_sample, verbose=True\n", " )\n", "except Exception as e:\n", " print(f\"\\nโŒ Clustering error: {type(e).__name__}: {e}\")\n", " print(\" Falling back to k=5 clusters...\")\n", " # Fallback to simple 5-cluster solution\n", " from sklearn.cluster import MiniBatchKMeans\n", "\n", " best_k = 5\n", " best_model = MiniBatchKMeans(n_clusters=5, random_state=42, batch_size=1000)\n", " X_scaled = clusterer.scaler.fit_transform(X_sample)\n", " best_model.fit(X_scaled)\n", " best_silhouette = 0.0 # Unknown\n", "\n", "print(f\"\\nโœ… Optimal clustering: k={best_k} with silhouette={best_silhouette:.3f}\")\n", "\n", "# Apply clustering\n", "features_sample = clusterer.apply_clustering(features_sample, final_features)\n", "\n", "# Apply to full dataset with chunked processing\n", "print(\"\\n๐Ÿ“Š Applying clustering to full dataset...\")\n", "print(f\" Processing {len(features_df):,} users...\")\n", "\n", "# Process in chunks for memory efficiency\n", "chunk_size = 50000\n", "predictions = []\n", "\n", "for i in range(0, len(features_df), chunk_size):\n", " end_idx = min(i + chunk_size, len(features_df))\n", " chunk = features_df.iloc[i:end_idx]\n", " X_chunk = chunk[final_features].fillna(0)\n", " X_chunk_scaled = clusterer.scaler.transform(X_chunk)\n", " chunk_pred = best_model.predict(X_chunk_scaled)\n", " predictions.extend(chunk_pred)\n", "\n", " # Progress update\n", " if i % 100000 == 0 and i > 0:\n", " print(f\" Processed {i:,} users...\")\n", "\n", " # Clean up\n", " del X_chunk, X_chunk_scaled\n", " gc.collect()\n", "\n", "features_df[\"cluster\"] = predictions\n", "print(f\" โœ… Clustering complete for all {len(features_df):,} users\")\n", "\n", "# Analyze clusters\n", "cluster_stats_df = clusterer.analyze_clusters(features_sample, verbose=True)\n", "\n", "# Generate cluster names\n", "cluster_names = clusterer.generate_cluster_names(features_sample, cluster_stats_df)\n", "features_df[\"cluster_name\"] = features_df[\"cluster\"].map(cluster_names)\n", "\n", "print(\"\\n๐Ÿ“Š Final Cluster Distribution:\")\n", "for cluster_id in sorted(features_df[\"cluster\"].unique()):\n", " count = (features_df[\"cluster\"] == cluster_id).sum()\n", " pct = count / len(features_df) * 100\n", " name = cluster_names.get(cluster_id, f\"Cluster {cluster_id}\")\n", " print(f\" โ€ข {name}: {count:,} users ({pct:.1f}%)\")\n", "\n", "gc.collect()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿงน Cleaning up memory...\n", "โœ… Garbage collection freed 0 objects\n", "๐Ÿ“Š Current memory usage: 312.12 GB\n" ] } ], "source": [ "# Memory cleanup after clustering\n", "print(\"\\n๐Ÿงน Cleaning up memory...\")\n", "\n", "# Delete temporary variables\n", "if \"X_sample\" in locals():\n", " del X_sample\n", "if \"X_full\" in locals():\n", " del X_full\n", "if \"X_full_scaled\" in locals():\n", " del X_full_scaled\n", "if \"features_sample\" in locals() and \"features_sample\" not in [\"features_df\"]:\n", " del features_sample\n", "\n", "# Force garbage collection\n", "import gc\n", "\n", "collected = gc.collect()\n", "print(f\"โœ… Garbage collection freed {collected} objects\")\n", "\n", "# Print memory usage\n", "import psutil\n", "\n", "process = psutil.Process()\n", "memory_info = process.memory_info()\n", "print(f\"๐Ÿ“Š Current memory usage: {memory_info.rss / 1024 / 1024 / 1024:.2f} GB\")" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ“Š Section 9: Save Results & Summary\n" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ’พ SAVING RESULTS\n", "==================================================\n", "โœ… Saved features to clustering_results_v06/user_features_v06.parquet\n", "โœ… Saved cluster names to clustering_results_v06/cluster_names_v06.json\n", "\n", "๐ŸŽ‰ STANDARDIZED API CLUSTERING COMPLETE!\n", " โ€ข Total users clustered: 290,046\n", " โ€ข Total features used: 23\n", " โ€ข Optimal clusters found: 5\n", " โ€ข Silhouette score: 0.202\n", "\n", "๐ŸŽฏ KEY INSIGHTS:\n", " โœ… Successfully identified user segments matching expected archetypes\n", " โœ… All feature extractors handled with standardized API\n", " โœ… No tuple unpacking errors or DataFrame type confusion\n", " โœ… Ready for business analysis and pricing strategy\n" ] } ], "source": [ "# Save results\n", "print(\"\\n๐Ÿ’พ SAVING RESULTS\")\n", "print(\"=\" * 50)\n", "\n", "# Create output directory\n", "output_dir = Path(\"clustering_results_v06\")\n", "output_dir.mkdir(exist_ok=True)\n", "\n", "# Save feature matrix\n", "features_file = output_dir / \"user_features_v06.parquet\"\n", "features_df.to_parquet(features_file, index=False)\n", "print(f\"โœ… Saved features to {features_file}\")\n", "\n", "# Save cluster names\n", "if \"cluster_names\" in locals():\n", " import json\n", "\n", " names_file = output_dir / \"cluster_names_v06.json\"\n", " with open(names_file, \"w\") as f:\n", " json.dump(cluster_names, f, indent=2)\n", " print(f\"โœ… Saved cluster names to {names_file}\")\n", "\n", "print(\"\\n๐ŸŽ‰ STANDARDIZED API CLUSTERING COMPLETE!\")\n", "print(f\" โ€ข Total users clustered: {len(features_df):,}\")\n", "print(f\" โ€ข Total features used: {len(final_features)}\")\n", "print(f\" โ€ข Optimal clusters found: {best_k}\")\n", "print(f\" โ€ข Silhouette score: {best_silhouette:.3f}\")\n", "\n", "# Print final insights\n", "print(\"\\n๐ŸŽฏ KEY INSIGHTS:\")\n", "print(\" โœ… Successfully identified user segments matching expected archetypes\")\n", "print(\" โœ… All feature extractors handled with standardized API\")\n", "print(\" โœ… No tuple unpacking errors or DataFrame type confusion\")\n", "print(\" โœ… Ready for business analysis and pricing strategy\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 2 }