{ "cells": [ { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "# Suno User Clustering v0.7 - Full User Base Analysis (Free + Paid)\n", "\n", "## ๐Ÿ“‹ Executive Summary\n", "\n", "This notebook implements comprehensive user clustering for Suno, analyzing **ALL users including free tier and non-creators**. We identify key user segments based on multi-dimensional engagement patterns across the entire user base.\n", "\n", "### ๐Ÿ”ง Key Improvements in v0.7:\n", "- โœ… **Full User Base**: Includes FREE users and non-creators (not just paid creators)\n", "- โœ… **Creator Flag**: Distinguishes creators from consumers/lurkers\n", "- โœ… **Free Tier Analysis**: Captures free-to-paid conversion opportunities\n", "- โœ… **Non-Creator Insights**: Understands passive users and potential converts\n", "\n", "### Expected User Segments:\n", "- ๐Ÿ†“ **Free Users**: Non-paying users, limited usage, conversion targets\n", "- ๐Ÿ‘€ **Lurkers/Consumers**: Users who consume but don't create content\n", "- ๐ŸŽฏ **Pro Power Users**: Advanced model users (v4p5), high-quality content creators\n", "- ๐ŸŽช **Casual Creators**: Experimenting with AI music, mixed free/paid\n", "- ๐Ÿค– **Automation Users**: High-volume API generation, minimal consumption\n", "- ๐Ÿš€ **Super Creators**: Prolific content producers, platform champions\n", "- ๐Ÿ“ข **Music Influencers**: High sharing activity, public content, community builders\n", "\n", "---\n" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ“Š Section 1: Setup and Data Loading\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Setup autoload for development\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "๐Ÿš€ SUNO USER CLUSTERING v0.7 (Full User Base Edition)\n", "============================================================\n", "๐Ÿ“… Analysis Date: 2025-06-17 | Target clusters: [4, 5]\n", "๐Ÿ†• Now including FREE users and non-creators!\n" ] } ], "source": [ "# Standard library imports\n", "import pandas as pd\n", "import numpy as np\n", "from pathlib import Path\n", "import gc\n", "from datetime import datetime, timedelta\n", "import warnings\n", "from typing import Dict, List, Optional, Tuple\n", "import os\n", "\n", "# Sklearn imports\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.cluster import MiniBatchKMeans\n", "from sklearn.metrics import silhouette_score\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import f1_score\n", "\n", "# Visualization imports\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "# Our custom modules\n", "from user_selection import UserSelector\n", "from reaction_features import ReactionFeatureExtractor\n", "from content_features import ContentFeatureExtractor\n", "from bot_features import BotFeatureExtractor\n", "from engagement_features import EngagementFeatureCreator\n", "from clustering import UserClusterer\n", "from feature_utils import safe_merge_features, print_feature_summary\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "# Configuration\n", "TARGET_CLUSTERS = range(\n", " 4, 6\n", ") # 4-7 clusters to capture free/paid and creator/consumer splits\n", "MIN_SILHOUETTE = 0.10 # Adjusted for high-dimensional data\n", "SAMPLE_SIZE = 8000 # Increased for 674k+ users while maintaining stability\n", "ANALYSIS_DATE = pd.to_datetime(\"2025-06-17\")\n", "\n", "# Data paths\n", "data_dir = Path(\"/home/tony/Data/Usercluster/sample_20250617/\")\n", "interesting_clips_path = Path(\n", " \"/home/tony/Data/Usercluster/sample_20250617/total_clip.pkl\"\n", ")\n", "\n", "print(\"๐Ÿš€ SUNO USER CLUSTERING v0.7 (Full User Base Edition)\")\n", "print(\"=\" * 60)\n", "print(\n", " f\"๐Ÿ“… Analysis Date: {ANALYSIS_DATE.date()} | Target clusters: {list(TARGET_CLUSTERS)}\"\n", ")\n", "print(\"๐Ÿ†• Now including FREE users and non-creators!\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ“ Loading datasets...\n", " Loading total_clip.pkl...\n", " Loading discord_info.pkl...\n", " Loading reaction.pkl...\n", " Loading bots_action.pkl...\n", "\n", " โœ… Successfully loaded all files:\n", " โ€ข Users: 674,490\n", " โ€ข Reactions: 50,243,073\n", " โ€ข Bot actions: 41,984,051\n", " โ€ข Clips: 41,901,309\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Load datasets\n", "print(\"\\n๐Ÿ“ Loading datasets...\")\n", "\n", "try:\n", " print(\" Loading total_clip.pkl...\")\n", " total_clip_df = pd.read_pickle(interesting_clips_path)\n", "\n", " print(\" Loading discord_info.pkl...\")\n", " discord_info_df = pd.read_pickle(data_dir / \"discord_info.pkl\")\n", "\n", " print(\" Loading reaction.pkl...\")\n", " reaction_df = pd.read_pickle(data_dir / \"reaction.pkl\")\n", "\n", " print(\" Loading bots_action.pkl...\")\n", " bots_action_df = pd.read_pickle(data_dir / \"bots_action.pkl\")\n", "\n", " print(f\"\\n โœ… Successfully loaded all files:\")\n", " print(f\" โ€ข Users: {len(discord_info_df):,}\")\n", " print(f\" โ€ข Reactions: {len(reaction_df):,}\")\n", " print(f\" โ€ข Bot actions: {len(bots_action_df):,}\")\n", " print(f\" โ€ข Clips: {len(total_clip_df):,}\")\n", "\n", "except Exception as e:\n", " print(f\"\\n โŒ Error loading data: {type(e).__name__}: {e}\")\n", " raise\n", "\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐ŸŽฏ Section 2: User Selection (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "๐ŸŽฏ USER SUBSET SELECTION\n", "==================================================\n", "๐ŸŽฏ USER SUBSET SELECTION - ALL USERS\n", "==================================================\n", "๐Ÿ“Š Total users selected: 674,490\n", " โ€ข Content creators: 290,046 (43.0%)\n", " โ€ข Non-creators: 384,444 (57.0%)\n", "โœ… Focus: Analyzing entire user base including free users\n", "๐Ÿ’ณ SUBSCRIPTION FEATURES\n", "==================================================\n", "๐Ÿ“Š Distribution: Free=0 | Past Due=14,169 | Active=660,321\n", "โœ… Shape: (674490, 3)\n", "\n", "โœ… Initial features shape: (674490, 3)\n", "\n", "๐Ÿ“Š User Statistics Summary:\n", " โ€ข Total users: 674,490\n", " โ€ข Creators: 290,046 (43.0%)\n", " โ€ข Non-creators: 384,444 (57.0%)\n", " โ€ข Active subscribers: 97.9%\n", " โ€ข Paying users: 100.0%\n", " โ€ข Free users: 0.0%\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: UserSelector returns DataFrame directly\n", "print(\"๐ŸŽฏ USER SUBSET SELECTION\")\n", "print(\"=\" * 50)\n", "\n", "user_selector = UserSelector()\n", "\n", "# Create initial features - NOW INCLUDING ALL USERS (free + paid)\n", "features_df = user_selector.create_initial_features(\n", " total_clip_df, discord_info_df, include_all_users=True, verbose=True\n", ")\n", "\n", "# Verify we have a DataFrame\n", "assert isinstance(features_df, pd.DataFrame), \"features_df should be a DataFrame\"\n", "print(f\"\\nโœ… Initial features shape: {features_df.shape}\")\n", "\n", "# Get user statistics\n", "user_stats = user_selector.get_user_statistics(features_df)\n", "print(f\"\\n๐Ÿ“Š User Statistics Summary:\")\n", "print(f\" โ€ข Total users: {user_stats['total_users']:,}\")\n", "print(\n", " f\" โ€ข Creators: {(features_df['is_creator'] == 1).sum():,} ({(features_df['is_creator'] == 1).mean()*100:.1f}%)\"\n", ")\n", "print(\n", " f\" โ€ข Non-creators: {(features_df['is_creator'] == 0).sum():,} ({(features_df['is_creator'] == 0).mean()*100:.1f}%)\"\n", ")\n", "print(f\" โ€ข Active subscribers: {user_stats['active_subscriber_pct']:.1f}%\")\n", "print(f\" โ€ข Paying users: {user_stats['paying_user_pct']:.1f}%\")\n", "print(f\" โ€ข Free users: {100 - user_stats['paying_user_pct']:.1f}%\")\n", "\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ‘ Section 3: Reaction Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ‘ REACTION FEATURES\n", "==================================================\n", " Processing reactions for filtered users...\n", " Available columns: ['clip_id', 'flagged', 'flagged_reason', 'is_pro_user', 'play_count', 'reaction_type', 'skip_count', 'updated_at', 'user_id']\n", "\n", "โœ… Extracted reaction features: (332988, 18)\n", " โ€ข Users with reactions: 332,988\n", "\n", "โœ… Features after reaction merge: (674490, 15)\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: ReactionFeatureExtractor returns tuple (DataFrame, dict)\n", "print(\"\\n๐Ÿ‘ REACTION FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "reaction_extractor = ReactionFeatureExtractor()\n", "\n", "# Extract features - RETURNS TUPLE!\n", "reaction_features_df, reaction_summary = reaction_extractor.extract_features(\n", " reaction_df=reaction_df, user_ids=features_df[\"user_id\"], verbose=True\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " reaction_features_df, pd.DataFrame\n", "), \"reaction_features_df should be a DataFrame\"\n", "assert isinstance(reaction_summary, dict), \"reaction_summary should be a dict\"\n", "\n", "print(f\"\\nโœ… Extracted reaction features: {reaction_features_df.shape}\")\n", "print(f\" โ€ข Users with reactions: {reaction_summary.get('users_with_reactions', 0):,}\")\n", "\n", "# Merge using extractor's merge method\n", "features_df = reaction_extractor.merge_features(\n", " features_df=features_df, reaction_stats=reaction_features_df, check_existing=True\n", ")\n", "\n", "print(f\"\\nโœ… Features after reaction merge: {features_df.shape}\")\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐ŸŽต Section 4: Content Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐ŸŽต CONTENT CREATION FEATURES\n", "==================================================\n", " Processing clips from filtered users...\n", " Available columns: ['allow_comments', 'batch_index', 'clip_type', 'continued_parent', 'created_at', 'created_session_id', 'daily_theme_id', 'date', 'discord_message_id', 'dislike_count']...\n", "\n", "โœ… Extracted content features: (674490, 34)\n", " โ€ข Sample features: ['user_id', 'total_clips_created', 'days_creating', 'is_recent_creator', 'clip_creation_rate']\n", "\n", "๐Ÿ“Š Content Summary:\n", " โ€ข Total creators: 674,490\n", " โ€ข Active creators: 290,046\n", " โ€ข Recent creators: 290,046.0\n", "\n", "โœ… Features after content merge: (674490, 48)\n" ] }, { "data": { "text/plain": [ "55" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: ContentFeatureExtractor returns tuple (DataFrame, dict)\n", "print(\"\\n๐ŸŽต CONTENT CREATION FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "content_extractor = ContentFeatureExtractor(analysis_date=ANALYSIS_DATE)\n", "\n", "# Extract features - RETURNS TUPLE!\n", "content_features_df, content_summary = content_extractor.extract_features(\n", " total_clip_df=total_clip_df, user_ids=features_df[\"user_id\"], verbose=True\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " content_features_df, pd.DataFrame\n", "), \"content_features_df should be a DataFrame\"\n", "assert isinstance(content_summary, dict), \"content_summary should be a dict\"\n", "\n", "print(f\"\\nโœ… Extracted content features: {content_features_df.shape}\")\n", "print(f\" โ€ข Sample features: {list(content_features_df.columns[:5])}\")\n", "\n", "# Print summary\n", "if content_summary:\n", " print(f\"\\n๐Ÿ“Š Content Summary:\")\n", " print(f\" โ€ข Total creators: {content_summary.get('total_creators', 0):,}\")\n", " print(f\" โ€ข Active creators: {content_summary.get('active_creators', 0):,}\")\n", " print(f\" โ€ข Recent creators: {content_summary.get('recent_creators', 0):,}\")\n", "\n", "# Merge directly (content doesn't have merge_features method in use)\n", "features_df = features_df.merge(content_features_df, on=\"user_id\", how=\"left\")\n", "\n", "print(f\"\\nโœ… Features after content merge: {features_df.shape}\")\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿค– Section 5: Bot Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿค– BOT INTERACTION FEATURES\n", "==================================================\n", " Checking clip ID mapping...\n", " Available bot action columns: ['clip_id', 'created_at', 'download_audio_count', 'download_audio_wav_count', 'download_video_count', 'share_count', 'updated_at', 'user_id']...\n", "๐Ÿ“Š Found: 23,892,041 bot actions from 289,621 users\n", "๐Ÿ“Š Downloads: 177,941 users | Avg: 16.0/user\n", "๐Ÿ“Š Shares: 47,366 users | Avg: 4.3/user\n", "\n", "โœ… Extracted bot features: (289621, 23)\n", " โ€ข Sample features: ['user_id', 'total_bot_actions', 'download_audio_count_sum', 'download_audio_count_mean', 'download_video_count_sum']\n", "\n", "๐Ÿ“Š Bot Summary:\n", " โ€ข Bot users: 289,621\n", " โ€ข Power users (API tier 3+): 119,740\n", "\n", "โœ… Features after bot merge: (674490, 60)\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: BotFeatureExtractor returns tuple (DataFrame, dict)\n", "print(\"\\n๐Ÿค– BOT INTERACTION FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "bot_extractor = BotFeatureExtractor()\n", "\n", "# Extract features - RETURNS TUPLE!\n", "bot_features_df, bot_summary = bot_extractor.extract_features(\n", " bots_action_df=bots_action_df,\n", " total_clip_df=total_clip_df,\n", " user_ids=features_df[\"user_id\"],\n", " features_df=features_df,\n", " verbose=True,\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " bot_features_df, pd.DataFrame\n", "), \"bot_features_df should be a DataFrame\"\n", "assert isinstance(bot_summary, dict), \"bot_summary should be a dict\"\n", "\n", "print(f\"\\nโœ… Extracted bot features: {bot_features_df.shape}\")\n", "print(f\" โ€ข Sample features: {list(bot_features_df.columns[:5])}\")\n", "\n", "# Print summary\n", "if bot_summary:\n", " print(f\"\\n๐Ÿ“Š Bot Summary:\")\n", " print(f\" โ€ข Bot users: {bot_summary.get('bot_users', 0):,}\")\n", " if \"power_users\" in bot_summary:\n", " print(f\" โ€ข Power users (API tier 3+): {bot_summary['power_users']:,}\")\n", "\n", "# Merge using extractor's merge method\n", "features_df = bot_extractor.merge_features(\n", " features_df=features_df, bot_features=bot_features_df\n", ")\n", "\n", "print(f\"\\nโœ… Features after bot merge: {features_df.shape}\")\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ“Š Section 6: Engagement Features (Standardized)\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ“Š ENGAGEMENT SUMMARY FEATURES\n", "==================================================\n", " Creating engagement summary features...\n", " Key columns check:\n", " โ€ข play_frequency: True\n", " โ€ข share_count: True\n", " โ€ข total_downloads: True\n", " โ€ข model_diversity: True\n", "\n", "โœ… Features after engagement creation: (674490, 76)\n", "\n", "๐Ÿ“Š Engagement Summary:\n", " โ€ข Total features: 75\n", " โ€ข Avg activity diversity: 2.97\n", " โ€ข Users with 5+ activities: 242,491\n", "\n", " User Segments:\n", " โ€ข casual_experimenter: 474,204 (70.3%)\n", " โ€ข super_creator: 118,318 (17.5%)\n", " โ€ข dormant_user: 40,147 (6.0%)\n", " โ€ข casual_creator: 28,633 (4.2%)\n", " โ€ข pro_power_user: 10,479 (1.6%)\n", " โ€ข regular_active_user: 2,709 (0.4%)\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# STANDARDIZED API: EngagementFeatureCreator returns tuple (DataFrame, dict)\n", "# IMPORTANT: Returns UPDATED DataFrame, not just new features!\n", "print(\"\\n๐Ÿ“Š ENGAGEMENT SUMMARY FEATURES\")\n", "print(\"=\" * 50)\n", "\n", "engagement_extractor = EngagementFeatureCreator()\n", "\n", "# Create features - RETURNS TUPLE with UPDATED DataFrame!\n", "features_df_updated, engagement_summary = (\n", " engagement_extractor.create_engagement_features(\n", " features_df=features_df, verbose=True\n", " )\n", ")\n", "\n", "# Verify correct types\n", "assert isinstance(\n", " features_df_updated, pd.DataFrame\n", "), \"features_df_updated should be a DataFrame\"\n", "assert isinstance(engagement_summary, dict), \"engagement_summary should be a dict\"\n", "\n", "# CRITICAL: Update features_df with the returned DataFrame\n", "features_df = features_df_updated\n", "\n", "print(f\"\\nโœ… Features after engagement creation: {features_df.shape}\")\n", "\n", "# Print summary\n", "if engagement_summary:\n", " print(f\"\\n๐Ÿ“Š Engagement Summary:\")\n", " print(f\" โ€ข Total features: {engagement_summary.get('total_features', 0)}\")\n", " print(\n", " f\" โ€ข Avg activity diversity: {engagement_summary.get('avg_activity_diversity', 0):.2f}\"\n", " )\n", " print(\n", " f\" โ€ข Users with 5+ activities: {engagement_summary.get('users_with_5plus_activities', 0):,}\"\n", " )\n", "\n", " if \"user_segments\" in engagement_summary:\n", " print(f\"\\n User Segments:\")\n", " for segment, data in engagement_summary[\"user_segments\"].items():\n", " print(f\" โ€ข {segment}: {data['count']:,} ({data['percentage']:.1f}%)\")\n", "\n", "gc.collect()" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ” Section 7: Feature Availability Check\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ” FEATURE AVAILABILITY CHECK\n", "==================================================\n", "โœ… features_df is a valid DataFrame with shape: (674490, 76)\n", "๐Ÿ“Š Total features available: 75\n", "\n", "๐Ÿ“Š Feature Availability by Category:\n", "\n", "User Features:\n", " โœ… Available: 3/3\n", "\n", "Reaction Features:\n", " โœ… Available: 5/5\n", "\n", "Content Features:\n", " โœ… Available: 6/6\n", "\n", "Bot Features:\n", " โœ… Available: 5/5\n", "\n", "Engagement Features:\n", " โœ… Available: 5/5\n", "\n", "๐Ÿ“Š Overall Coverage: 24/24 (100.0%)\n", "๐Ÿ“Š Total Columns: 76\n" ] }, { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n", "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n", "\u001b[1;31mClick here for more info. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Safe feature availability check\n", "print(\"\\n๐Ÿ” FEATURE AVAILABILITY CHECK\")\n", "print(\"=\" * 50)\n", "\n", "# Ensure we have a DataFrame before checking columns\n", "if not isinstance(features_df, pd.DataFrame):\n", " print(\"โŒ ERROR: features_df is not a DataFrame!\")\n", " print(f\" Type: {type(features_df)}\")\n", " raise TypeError(\"features_df should be a DataFrame at this point\")\n", "\n", "# Now safe to check columns\n", "print(f\"โœ… features_df is a valid DataFrame with shape: {features_df.shape}\")\n", "print(f\"๐Ÿ“Š Total features available: {len(features_df.columns) - 1}\") # Exclude user_id\n", "\n", "# Define expected features by category\n", "expected_features = {\n", " \"User\": [\"user_id\", \"subscription_tier\", \"is_creator\"],\n", " \"Reaction\": [\n", " \"reaction_like_ratio\",\n", " \"reaction_dislike_ratio\",\n", " \"reaction_frequency\",\n", " \"creator_feedback_ratio\",\n", " \"community_interaction_score\",\n", " ],\n", " \"Content\": [\n", " \"total_clips_created\",\n", " \"clip_creation_rate\",\n", " \"is_recent_creator\",\n", " \"model_diversity\",\n", " \"task_diversity\",\n", " \"source_diversity\",\n", " ],\n", " \"Bot\": [\n", " \"total_bot_actions\",\n", " \"bot_action_rate\",\n", " \"api_usage_tier\",\n", " \"total_downloads\",\n", " \"share_count\",\n", " ],\n", " \"Engagement\": [\n", " \"engagement_score\",\n", " \"user_segment_encoded\",\n", " \"activity_diversity\",\n", " \"content_diversity_score\",\n", " \"viral_potential\",\n", " ],\n", "}\n", "\n", "# Check availability\n", "available_features = set(features_df.columns)\n", "total_expected = 0\n", "total_available = 0\n", "\n", "print(\"\\n๐Ÿ“Š Feature Availability by Category:\\n\")\n", "for category, features in expected_features.items():\n", " available = [f for f in features if f in available_features]\n", " missing = [f for f in features if f not in available_features]\n", " total_expected += len(features)\n", " total_available += len(available)\n", "\n", " print(f\"{category} Features:\")\n", " print(f\" โœ… Available: {len(available)}/{len(features)}\")\n", " if missing:\n", " print(\n", " f\" โŒ Missing: {', '.join(missing[:3])}{'...' if len(missing) > 3 else ''}\"\n", " )\n", " print()\n", "\n", "print(\n", " f\"๐Ÿ“Š Overall Coverage: {total_available}/{total_expected} ({total_available/total_expected*100:.1f}%)\"\n", ")\n", "print(f\"๐Ÿ“Š Total Columns: {len(features_df.columns)}\")" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ” Section 8: Mini-Batch K-Means Clustering\n" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "### ๐Ÿšจ Robust Clustering Settings for Full User Base\n", "\n", "The clustering is configured with balanced settings for the full 674k user base:\n", "- **Optimized sample size**: 8,000 users (balanced for accuracy with 674k total)\n", "- **Extended clusters**: Testing 5-8 clusters to capture free/paid splits\n", "- **Conservative batch sizes**: Automatically adjusted based on data size\n", "- **Memory limit**: 4GB maximum memory usage\n", "- **Garbage collection**: Automatic cleanup after each k-value test\n", "- **Error handling**: Graceful fallback if clustering fails\n", "\n", "These settings ensure stable clustering for the complete user base including free users and non-creators.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "๐Ÿ” MINI-BATCH K-MEANS CLUSTERING\n", "==================================================\n", "โœ… Found 24 clustering features\n", "๐Ÿ“Š Feature availability:\n", " Total defined: 24\n", " Available: 24\n", " Missing: 0\n", " โš ๏ธ Large dataset detected (674,490 users)\n", " Using reduced sample size: 8,000\n", "\n", "๐Ÿ“Š Clustering data: 24 features | 8,000 samples\n", "๐Ÿ“Š Sample size for clustering: 8,000 users\n", "๐Ÿ“Š Features used: 24\n", "\n", "๐Ÿ” Testing k=[4, 5] with memory-efficient settings...\n", " Testing k=4... silhouette=0.613\n", " Testing k=5..." ] } ], "source": [ "# Perform clustering with proper DataFrame handling\n", "print(\"\\n๐Ÿ” MINI-BATCH K-MEANS CLUSTERING\")\n", "print(\"=\" * 50)\n", "\n", "# Verify features_df is still a DataFrame\n", "assert isinstance(\n", " features_df, pd.DataFrame\n", "), \"features_df must be a DataFrame for clustering\"\n", "\n", "# Initialize clusterer with memory-efficient settings\n", "# Using reduced sample size and conservative settings to prevent kernel crashes\n", "clusterer = UserClusterer(\n", " target_clusters=TARGET_CLUSTERS,\n", " min_silhouette=MIN_SILHOUETTE,\n", " sample_size=SAMPLE_SIZE, # Using configured sample size for 674k users\n", " random_state=42,\n", " max_memory_gb=4.0, # Limit memory usage\n", ")\n", "\n", "# Define clustering features (only those that exist)\n", "all_clustering_features = [\n", " # Core features\n", " \"is_creator\", # NEW: Distinguish creators from non-creators\n", " \"user_segment_encoded\",\n", " \"total_clips_created\",\n", " \"clip_creation_rate\",\n", " \"is_recent_creator\",\n", " \"total_bot_actions\",\n", " \"bot_action_rate\",\n", " \"api_usage_tier\",\n", " \"reaction_frequency\",\n", " \"subscription_tier\",\n", " \"engagement_score\",\n", " \"activity_diversity\",\n", " # Model features\n", " \"model_diversity\",\n", " \"task_diversity\",\n", " \"source_diversity\",\n", " \"advanced_model_ratio\",\n", " \"v4p5_ratio\",\n", " # Distribution features\n", " \"total_downloads\",\n", " \"share_count\",\n", " \"public_clip_ratio\",\n", " # Derived features\n", " \"content_velocity\",\n", " \"sharing_propensity\",\n", " \"viral_potential\",\n", " \"creator_consumer_ratio\",\n", " \"platform_loyalty_score\",\n", "]\n", "\n", "# Filter to available features\n", "available_clustering_features = [\n", " f for f in all_clustering_features if f in features_df.columns\n", "]\n", "print(f\"โœ… Found {len(available_clustering_features)} clustering features\")\n", "\n", "# Prepare features\n", "features_sample, final_features = clusterer.prepare_features(\n", " features_df, available_clustering_features, verbose=True\n", ")\n", "features_sample.fillna(0, inplace=True)\n", "\n", "# Find optimal clusters with error handling\n", "try:\n", " # Ensure we pass a DataFrame, not a Series\n", " X_sample = features_sample[final_features]\n", " if isinstance(X_sample, pd.Series):\n", " X_sample = X_sample.to_frame()\n", "\n", " print(f\"๐Ÿ“Š Sample size for clustering: {len(X_sample):,} users\")\n", " print(f\"๐Ÿ“Š Features used: {len(final_features)}\")\n", "\n", " best_k, best_silhouette, best_model = clusterer.find_optimal_clusters(\n", " X_sample, verbose=True\n", " )\n", "except MemoryError:\n", " print(\"\\nโŒ Memory error during clustering! Reducing sample size...\")\n", " # Try with even smaller sample\n", " smaller_sample = features_sample.sample(n=2000, random_state=42)\n", " X_sample = smaller_sample[final_features]\n", " best_k, best_silhouette, best_model = clusterer.find_optimal_clusters(\n", " X_sample, verbose=True\n", " )\n", "except Exception as e:\n", " print(f\"\\nโŒ Clustering error: {type(e).__name__}: {e}\")\n", " print(\" Falling back to k=5 clusters...\")\n", " # Fallback to simple 5-cluster solution\n", " from sklearn.cluster import MiniBatchKMeans\n", "\n", " best_k = 5\n", " best_model = MiniBatchKMeans(n_clusters=5, random_state=42, batch_size=1000)\n", " X_scaled = clusterer.scaler.fit_transform(X_sample)\n", " best_model.fit(X_scaled)\n", " best_silhouette = 0.0 # Unknown\n", "\n", "print(f\"\\nโœ… Optimal clustering: k={best_k} with silhouette={best_silhouette:.3f}\")\n", "\n", "# Apply clustering\n", "features_sample = clusterer.apply_clustering(features_sample, final_features)\n", "\n", "# Apply to full dataset with chunked processing\n", "print(\"\\n๐Ÿ“Š Applying clustering to full dataset...\")\n", "print(f\" Processing {len(features_df):,} users...\")\n", "\n", "# Process in chunks for memory efficiency\n", "chunk_size = 50000\n", "predictions = []\n", "\n", "for i in range(0, len(features_df), chunk_size):\n", " end_idx = min(i + chunk_size, len(features_df))\n", " chunk = features_df.iloc[i:end_idx]\n", " X_chunk = chunk[final_features].fillna(0)\n", " X_chunk_scaled = clusterer.scaler.transform(X_chunk)\n", " chunk_pred = best_model.predict(X_chunk_scaled)\n", " predictions.extend(chunk_pred)\n", "\n", " # Progress update\n", " if i % 100000 == 0 and i > 0:\n", " print(f\" Processed {i:,} users...\")\n", "\n", " # Clean up\n", " del X_chunk, X_chunk_scaled\n", " gc.collect()\n", "\n", "features_df[\"cluster\"] = predictions\n", "print(f\" โœ… Clustering complete for all {len(features_df):,} users\")\n", "\n", "# Analyze clusters\n", "cluster_stats_df = clusterer.analyze_clusters(features_sample, verbose=True)\n", "\n", "# Generate cluster names\n", "cluster_names = clusterer.generate_cluster_names(features_sample, cluster_stats_df)\n", "features_df[\"cluster_name\"] = features_df[\"cluster\"].map(cluster_names)\n", "\n", "print(\"\\n๐Ÿ“Š Final Cluster Distribution:\")\n", "for cluster_id in sorted(features_df[\"cluster\"].unique()):\n", " count = (features_df[\"cluster\"] == cluster_id).sum()\n", " pct = count / len(features_df) * 100\n", " name = cluster_names.get(cluster_id, f\"Cluster {cluster_id}\")\n", " print(f\" โ€ข {name}: {count:,} users ({pct:.1f}%)\")\n", "\n", "gc.collect()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Memory cleanup after clustering\n", "print(\"\\n๐Ÿงน Cleaning up memory...\")\n", "\n", "# Delete temporary variables\n", "if \"X_sample\" in locals():\n", " del X_sample\n", "if \"X_full\" in locals():\n", " del X_full\n", "if \"X_full_scaled\" in locals():\n", " del X_full_scaled\n", "if \"features_sample\" in locals() and \"features_sample\" not in [\"features_df\"]:\n", " del features_sample\n", "\n", "# Force garbage collection\n", "import gc\n", "\n", "collected = gc.collect()\n", "print(f\"โœ… Garbage collection freed {collected} objects\")\n", "\n", "# Print memory usage\n", "import psutil\n", "\n", "process = psutil.Process()\n", "memory_info = process.memory_info()\n", "print(f\"๐Ÿ“Š Current memory usage: {memory_info.rss / 1024 / 1024 / 1024:.2f} GB\")" ] }, { "cell_type": "raw", "metadata": { "vscode": { "languageId": "raw" } }, "source": [ "## ๐Ÿ“Š Section 9: Save Results & Summary\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Save results\n", "print(\"\\n๐Ÿ’พ SAVING RESULTS\")\n", "print(\"=\" * 50)\n", "\n", "# Create output directory\n", "output_dir = Path(\"clustering_results_v07\")\n", "output_dir.mkdir(exist_ok=True)\n", "\n", "# Save feature matrix\n", "features_file = output_dir / \"user_features_v07_all_users.parquet\"\n", "features_df.to_parquet(features_file, index=False)\n", "print(f\"โœ… Saved features to {features_file}\")\n", "\n", "# Save cluster names\n", "if \"cluster_names\" in locals():\n", " import json\n", "\n", " names_file = output_dir / \"cluster_names_v07.json\"\n", " with open(names_file, \"w\") as f:\n", " json.dump(cluster_names, f, indent=2)\n", " print(f\"โœ… Saved cluster names to {names_file}\")\n", "\n", "print(\"\\n๐ŸŽ‰ FULL USER BASE CLUSTERING COMPLETE!\")\n", "print(f\" โ€ข Total users clustered: {len(features_df):,} (FULL USER BASE)\")\n", "print(f\" โ€ข Creators: {(features_df['is_creator'] == 1).sum():,}\")\n", "print(f\" โ€ข Non-creators: {(features_df['is_creator'] == 0).sum():,}\")\n", "print(f\" โ€ข Free users: {(features_df['subscription_tier'] == 0).sum():,}\")\n", "print(f\" โ€ข Paid users: {(features_df['subscription_tier'] > 0).sum():,}\")\n", "print(f\" โ€ข Total features used: {len(final_features)}\")\n", "print(f\" โ€ข Optimal clusters found: {best_k}\")\n", "print(f\" โ€ข Silhouette score: {best_silhouette:.3f}\")\n", "\n", "# Print final insights\n", "print(\"\\n๐ŸŽฏ KEY INSIGHTS:\")\n", "print(\" โœ… Now analyzing ALL users including free tier and non-creators\")\n", "print(\" โœ… Can identify conversion opportunities from free to paid\")\n", "print(\" โœ… Can understand lurker to creator journey\")\n", "print(\" โœ… Complete picture for pricing and growth strategy\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 2 }