{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "0", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "from suno_utils.utils.text import read_jsonl, read_json, write_json, write_jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "1", "metadata": {}, "outputs": [], "source": [ "test_key = \"splice_f20c6eaf407d9048e8ad1b5307296a4ee6954ed6632468a7f5fbdf3294e2fe33.opus\"\n", "test_path = \"/app2/suno/data/sara/sfx_get_beats/opus_audio/\" + test_key" ] }, { "cell_type": "code", "execution_count": null, "id": "2", "metadata": {}, "outputs": [], "source": [ "test = Audio.from_file(test_path)\n", "test.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "3", "metadata": {}, "outputs": [], "source": [ "test_trim = \"splice_2e382f24f15d611c3d55f50ffd5a9dddfff13ae716087c52399f73256da8bfc1.opus\"\n", "test_path_trim = \"/app2/suno/data/sara/sfx_get_beats/opus_audio_truncated/\" + test_trim\n", "test_trim = Audio.from_file(test_path_trim)\n", "test_trim.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5", "metadata": {}, "outputs": [], "source": [ "sfx_metas = read_jsonl(\"/app2/suno/data/sara/sfx_get_beats/combined_v3_w_extreme_metas_v0.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6", "metadata": {}, "outputs": [], "source": [ "import os\n", "from tqdm import tqdm\n", "\n", "opus_path = \"/app2/suno/data/sara/sfx_get_beats/opus_audio\"\n", "opus_truncated_path = \"/app2/suno/data/sara/sfx_get_beats/opus_audio_truncated\"\n", "missing = []\n", "\n", "for meta in tqdm(sfx_metas):\n", " song_id = meta[\"id\"]\n", " opus_filename = song_id + \".opus\"\n", " \n", " opus_cand = os.path.join(opus_path, opus_filename)\n", " trunc_cand = os.path.join(opus_truncated_path, opus_filename)\n", " if os.path.exists(trunc_cand):\n", " meta[\"s3_filepath\"] = trunc_cand\n", " elif os.path.exists(opus_cand):\n", " meta[\"s3_filepath\"] = opus_cand\n", " else:\n", " missing.append(song_id)\n", "\n", "print(len(missing))" ] }, { "cell_type": "code", "execution_count": null, "id": "7", "metadata": {}, "outputs": [], "source": [ "youtube_hits = read_json(\"youtube_results.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "8", "metadata": {}, "outputs": [], "source": [ "youtube_hits[0].keys()" ] }, { "cell_type": "code", "execution_count": null, "id": "9", "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": null, "id": "10", "metadata": {}, "outputs": [], "source": [ "yt_df = pd.read_csv(\"youtube_results.csv\")\n", "yt_df.head()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "11", "metadata": {}, "outputs": [], "source": [ "len(yt_df)" ] }, { "cell_type": "code", "execution_count": null, "id": "12", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", "# Filter out None values and convert to numeric\n", "match_scores = yt_df['Match Score'].dropna()\n", "\n", "# Create histogram\n", "plt.figure(figsize=(10, 6))\n", "plt.hist(match_scores, bins=50, edgecolor='black', alpha=0.7)\n", "plt.xlabel('Match Score')\n", "plt.ylabel('Frequency')\n", "plt.title(f'Distribution of Match Score (n={len(match_scores):,}, excluded {yt_df[\"Match Score\"].isna().sum()} None values)')\n", "plt.grid(True, alpha=0.3)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "# Print some statistics\n", "print(f\"\\nMatch Score Statistics:\")\n", "print(f\"Total records: {len(yt_df)}\")\n", "print(f\"Non-null values: {len(match_scores)}\")\n", "print(f\"Null/None values: {yt_df['Match Score'].isna().sum()}\")\n", "print(f\"\\nMin: {match_scores.min():.4f}\")\n", "print(f\"Max: {match_scores.max():.4f}\")\n", "print(f\"Mean: {match_scores.mean():.4f}\")\n", "print(f\"Median: {match_scores.median():.4f}\")\n", "print(f\"Std Dev: {match_scores.std():.4f}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "13", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "14", "metadata": {}, "outputs": [], "source": [ "yt_df_sorted.to_csv(\"youtube_results_found.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "15", "metadata": {}, "outputs": [], "source": [ "for k,v in yt_df_sorted.iloc[2].items():\n", " print(k,v)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "16", "metadata": {}, "outputs": [], "source": [ "# Filter out None values and sort by Match Score ascending (lowest first)\n", "yt_df_sorted = yt_df[yt_df['Match Score'].notna()].sort_values('Match Score', ascending=False)\n", "\n", "# Show the bottom 20 (lowest scores)\n", "print(len(yt_df_sorted))\n", "yt_df_sorted.head(20)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "17", "metadata": {}, "outputs": [], "source": [ "cleaned_mashups_discogs_s3 = read_jsonl(\"/home/sara/task_data/cleaned_mashup_data_wout_ws_discogs_s3.jsonl\")\n", "cleaned_mashups_discogs_s3[0].keys()\n", "cleaned_df_discogs_s3 = pd.DataFrame(cleaned_mashups_discogs_s3)\n", "print(len(cleaned_df_discogs_s3))\n", "cleaned_df_discogs_s3[cleaned_df_discogs_s3[\"s3_filepath\"].notna()].head()" ] }, { "cell_type": "code", "execution_count": null, "id": "18", "metadata": {}, "outputs": [], "source": [ "has_s3 = cleaned_df_discogs_s3[cleaned_df_discogs_s3[\"s3_filepath\"].notna()]\n", "just_strong_w_s3 = has_s3[has_s3['mashup_strength'] == 'very strong']\n", "len(just_strong_w_s3)" ] }, { "cell_type": "code", "execution_count": null, "id": "19", "metadata": {}, "outputs": [], "source": [ "for x,y in just_strong_w_s3.iloc[0].items():\n", " print(x,y)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "20", "metadata": {}, "outputs": [], "source": [ "json_yt = read_json(\"youtube_results.json\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "21", "metadata": {}, "outputs": [], "source": [ "merged_data = []\n", "for m in json_yt:\n", " og_row = m['original_data'].copy()\n", " yt_match = m['youtube_match']\n", " \n", " if yt_match is not None:\n", " og_row['yt_video_title'] = yt_match['title']\n", " og_row['yt_channel_title'] = yt_match['channel']\n", " og_row['yt_match_score'] = yt_match['match_score']\n", " og_row['yt_duration'] = yt_match['duration']\n", " og_row['yt_url'] = yt_match['url']\n", " og_row['yt_views'] = yt_match['views']\n", " else:\n", " og_row['yt_video_title'] = None\n", " og_row['yt_channel_title'] = None\n", " og_row['yt_match_score'] = None\n", " og_row['yt_duration'] = None\n", " og_row['yt_url'] = None\n", " og_row['yt_views'] = None\n", "\n", " merged_data.append(og_row)\n", "\n", "print(len(merged_data))" ] }, { "cell_type": "code", "execution_count": null, "id": "22", "metadata": {}, "outputs": [], "source": [ "merged_data[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "23", "metadata": {}, "outputs": [], "source": [ "write_jsonl(merged_data, \"/home/sara/task_data/cleaned_mashup_data_wout_ws_w_yt_data.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "24", "metadata": {}, "outputs": [], "source": [ "ws_mashups = read_jsonl(\"/home/sara/task_data/final_whosampled_samples_11_12.jsonl\")\n", "print(ws_mashups[0].keys())\n", "ws_mashups_df = pd.DataFrame(ws_mashups)\n", "ws_mashups_df.head()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "25", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", "# Create figure with subplots\n", "fig, axes = plt.subplots(2, 2, figsize=(15, 12))\n", "fig.suptitle('Distribution Statistics for Song Mashup Data', fontsize=16, fontweight='bold')\n", "\n", "# 1. Distribution of source_vote\n", "ax1 = axes[0, 0]\n", "source_votes = ws_mashups_df['source_vote'].dropna()\n", "# Limit to 95th percentile to focus on main distribution\n", "xlim_max = source_votes.quantile(0.95)\n", "ax1.hist(source_votes[source_votes <= xlim_max], bins=50, edgecolor='black', alpha=0.7, color='steelblue')\n", "ax1.set_xlabel('Source Vote')\n", "ax1.set_ylabel('Frequency')\n", "ax1.set_xlim(0, xlim_max)\n", "ax1.set_title(f'Distribution of Source Vote\\n(n={len(source_votes):,}, mean={source_votes.mean():.2f}, median={source_votes.median():.2f}, x-axis: 0-{int(xlim_max)})')\n", "ax1.grid(True, alpha=0.3)\n", "\n", "# 2. Distribution of duration\n", "ax2 = axes[0, 1]\n", "durations = ws_mashups_df['duration'].dropna()\n", "# Limit to 95th percentile to focus on main distribution (exclude extreme outliers)\n", "xlim_max = durations.quantile(0.95)\n", "ax2.hist(durations[durations <= xlim_max], bins=50, edgecolor='black', alpha=0.7, color='coral')\n", "ax2.set_xlabel('Duration (seconds)')\n", "ax2.set_ylabel('Frequency')\n", "ax2.set_xlim(0, xlim_max)\n", "ax2.set_title(f'Distribution of Duration\\n(n={len(durations):,}, mean={durations.mean():.2f}s, median={durations.median():.2f}s, x-axis: 0-{int(xlim_max)}s)')\n", "ax2.grid(True, alpha=0.3)\n", "\n", "# 3. Distribution of release_date\n", "ax3 = axes[1, 0]\n", "release_dates = ws_mashups_df['release_date'].dropna()\n", "release_dates = release_dates[release_dates != 0] # Filter out 0 values (empty)\n", "# Focus on reasonable year range (1950-2025) to exclude outliers\n", "release_dates_filtered = release_dates[(release_dates >= 1950) & (release_dates <= 2025)]\n", "ax3.hist(release_dates_filtered, bins=50, edgecolor='black', alpha=0.7, color='mediumseagreen')\n", "ax3.set_xlabel('Release Date (year)')\n", "ax3.set_ylabel('Frequency')\n", "ax3.set_xlim(1950, 2025)\n", "ax3.set_title(f'Distribution of Release Date\\n(n={len(release_dates_filtered):,}, mean={release_dates_filtered.mean():.0f}, median={release_dates_filtered.median():.0f}, x-axis: 1950-2025)')\n", "ax3.grid(True, alpha=0.3)\n", "\n", "# 4. Distribution of number of artists\n", "ax4 = axes[1, 1]\n", "# Calculate number of artists for each row\n", "num_artists = ws_mashups_df['artists'].apply(lambda x: len(x) if isinstance(x, list) else 0)\n", "ax4.hist(num_artists, bins=range(int(num_artists.max())+2), edgecolor='black', alpha=0.7, color='plum')\n", "ax4.set_xlabel('Number of Artists')\n", "ax4.set_ylabel('Frequency')\n", "ax4.set_title(f'Distribution of Number of Artists\\n(n={len(num_artists):,}, mean={num_artists.mean():.2f}, median={num_artists.median():.2f})')\n", "ax4.grid(True, alpha=0.3)\n", "ax4.set_xticks(range(int(num_artists.max())+1))\n", "\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "# Print summary statistics\n", "print(\"=\" * 60)\n", "print(\"SUMMARY STATISTICS\")\n", "print(\"=\" * 60)\n", "print(f\"\\nTotal rows: {len(ws_mashups_df):,}\")\n", "\n", "# Calculate total hours\n", "total_duration_seconds = ws_mashups_df['duration'].sum()\n", "total_hours = total_duration_seconds / 3600\n", "print(f\"Total duration: {total_duration_seconds:,.2f} seconds\")\n", "print(f\"Total duration: {total_hours:,.2f} hours\")\n", "print(f\"Total duration: {total_hours/24:,.2f} days\")\n", "\n", "print(\"\\n\" + \"=\" * 60)\n", "print(\"DETAILED STATISTICS BY FIELD\")\n", "print(\"=\" * 60)\n", "\n", "print(f\"\\nSource Vote:\")\n", "print(f\" Non-null values: {ws_mashups_df['source_vote'].notna().sum():,}\")\n", "print(f\" Null values: {ws_mashups_df['source_vote'].isna().sum():,}\")\n", "if len(source_votes) > 0:\n", " print(f\" Min: {source_votes.min()}\")\n", " print(f\" Max: {source_votes.max()}\")\n", " print(f\" Mean: {source_votes.mean():.2f}\")\n", " print(f\" Median: {source_votes.median():.2f}\")\n", " print(f\" Std Dev: {source_votes.std():.2f}\")\n", "\n", "print(f\"\\nDuration:\")\n", "print(f\" Non-null values: {ws_mashups_df['duration'].notna().sum():,}\")\n", "print(f\" Null values: {ws_mashups_df['duration'].isna().sum():,}\")\n", "if len(durations) > 0:\n", " print(f\" Min: {durations.min():.2f} seconds\")\n", " print(f\" Max: {durations.max():.2f} seconds\")\n", " print(f\" Mean: {durations.mean():.2f} seconds\")\n", " print(f\" Median: {durations.median():.2f} seconds\")\n", " print(f\" Std Dev: {durations.std():.2f} seconds\")\n", "\n", "print(f\"\\nRelease Date:\")\n", "print(f\" Non-null values: {ws_mashups_df['release_date'].notna().sum():,}\")\n", "print(f\" Null values: {ws_mashups_df['release_date'].isna().sum():,}\")\n", "print(f\" Zero values (excluded): {(ws_mashups_df['release_date'] == 0).sum():,}\")\n", "if len(release_dates) > 0:\n", " print(f\" Min: {int(release_dates.min())}\")\n", " print(f\" Max: {int(release_dates.max())}\")\n", " print(f\" Mean: {release_dates.mean():.0f}\")\n", " print(f\" Median: {release_dates.median():.0f}\")\n", " print(f\" Std Dev: {release_dates.std():.0f}\")\n", "\n", "print(f\"\\nNumber of Artists:\")\n", "print(f\" Mean: {num_artists.mean():.2f}\")\n", "print(f\" Median: {num_artists.median():.2f}\")\n", "print(f\" Min: {int(num_artists.min())}\")\n", "print(f\" Max: {int(num_artists.max())}\")\n", "print(f\" Std Dev: {num_artists.std():.2f}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "26", "metadata": {}, "outputs": [], "source": [ "ws_mashups = read_jsonl(\"/home/sara/task_data/final_whosampled_mashups_11_12.jsonl\")\n", "print(ws_mashups[0].keys())\n", "ws_mashups_df = pd.DataFrame(ws_mashups)\n", "ws_mashups_df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "27", "metadata": {}, "outputs": [], "source": [ "ws_mashups_df['key'] = None\n", "ws_mashups_df['bpm'] = None\n", "ws_mashups_df['genre'] = None\n", "ws_mashups_df['label'] = None\n", "ws_mashups_df['s3_filepath'] = None\n", "ws_mashups_df['mashup_strength'] = 'strong'\n", "ws_mashups_df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "28", "metadata": {}, "outputs": [], "source": [ "other_mashups = read_jsonl(\"/home/sara/task_data/cleaned_mashup_data_wout_ws_discogs_s3.jsonl\")\n", "print(other_mashups[0].keys())\n", "print(len(other_mashups))\n", "other_mashups_df = pd.DataFrame(other_mashups)\n", "other_mashups_df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "29", "metadata": {}, "outputs": [], "source": [ "other_mashups_df.drop(columns=['delimiter_found', 'mashup_keyword', 'metadata', 'split_parts'], inplace=True)\n", "other_mashups_df['output_url'] = None\n", "other_mashups_df['source_urls'] = None\n", "other_mashups_df['source_votes'] = None\n", "other_mashups_df['source_ids'] = None\n", "other_mashups_df['output_id'] = None\n", "other_mashups_df.head()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "30", "metadata": {}, "outputs": [], "source": [ "all_mashups = pd.concat([ws_mashups_df, other_mashups_df])\n", "all_mashups.head()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "31", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "def extract_year_simple(date_val):\n", " if pd.isna(date_val) or date_val is None or date_val == 0:\n", " return None if date_val == 0 else date_val # Convert 0 to None, keep None as None\n", " \n", " date_str = str(date_val)\n", " \n", " # If already a 4-digit year, return as int\n", " if len(date_str) == 4 and date_str.isdigit():\n", " return int(date_str)\n", " \n", " # Otherwise, take first 4 characters if they're digits\n", " if len(date_str) >= 4 and date_str[:4].isdigit():\n", " return int(date_str[:4])\n", " \n", " return None\n", "\n", "# Reformat release_date to be just the year\n", "all_mashups['release_date'] = all_mashups['release_date'].apply(extract_year_simple)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "32", "metadata": {}, "outputs": [], "source": [ "all_mashups.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "33", "metadata": {}, "outputs": [], "source": [ "all_mashups.to_json('/home/sara/task_data/all_consolidated_mashups_11_13_missing_some_yt.jsonl', orient='records', lines=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "34", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", "# Create figure with subplots\n", "fig, axes = plt.subplots(2, 2, figsize=(15, 12))\n", "fig.suptitle('Distribution Statistics for All Mashups Data', fontsize=16, fontweight='bold')\n", "\n", "# 1. Distribution of release_date (ignore None)\n", "ax1 = axes[0, 0]\n", "release_dates = all_mashups['release_date'].dropna()\n", "release_dates = release_dates[release_dates != 0] # Filter out 0 values (empty)\n", "# Focus on reasonable year range (1950-2025) to exclude outliers\n", "release_dates_filtered = release_dates[(release_dates >= 1950) & (release_dates <= 2025)]\n", "ax1.hist(release_dates_filtered, bins=50, edgecolor='black', alpha=0.7, color='steelblue')\n", "ax1.set_xlabel('Release Date (year)')\n", "ax1.set_ylabel('Frequency')\n", "ax1.set_xlim(1950, 2025)\n", "ax1.set_title(f'Distribution of Release Date\\n(n={len(release_dates_filtered):,}, mean={release_dates_filtered.mean():.0f}, median={release_dates_filtered.median():.0f}, x-axis: 1950-2025)')\n", "ax1.grid(True, alpha=0.3)\n", "\n", "# 2. Distribution of duration (ignore None)\n", "ax2 = axes[0, 1]\n", "durations = all_mashups['duration'].dropna()\n", "# Limit to 95th percentile to focus on main distribution (exclude extreme outliers)\n", "xlim_max = durations.quantile(0.95)\n", "ax2.hist(durations[durations <= xlim_max], bins=50, edgecolor='black', alpha=0.7, color='coral')\n", "ax2.set_xlabel('Duration (seconds)')\n", "ax2.set_ylabel('Frequency')\n", "ax2.set_xlim(0, xlim_max)\n", "ax2.set_title(f'Distribution of Duration\\n(n={len(durations):,}, mean={durations.mean():.2f}s, median={durations.median():.2f}s, x-axis: 0-{int(xlim_max)}s)')\n", "ax2.grid(True, alpha=0.3)\n", "\n", "# 3. Distribution of mashup_strength\n", "ax3 = axes[1, 0]\n", "mashup_strength = all_mashups['mashup_strength'].dropna()\n", "strength_counts = mashup_strength.value_counts()\n", "ax3.bar(range(len(strength_counts)), strength_counts.values, edgecolor='black', alpha=0.7, color='mediumseagreen')\n", "ax3.set_xticks(range(len(strength_counts)))\n", "ax3.set_xticklabels(strength_counts.index, rotation=45, ha='right')\n", "ax3.set_xlabel('Mashup Strength')\n", "ax3.set_ylabel('Frequency')\n", "ax3.set_title(f'Distribution of Mashup Strength\\n(n={len(mashup_strength):,})')\n", "ax3.grid(True, alpha=0.3, axis='y')\n", "\n", "# 4. Distribution of data_source\n", "ax4 = axes[1, 1]\n", "data_source = all_mashups['data_source'].dropna()\n", "source_counts = data_source.value_counts()\n", "ax4.bar(range(len(source_counts)), source_counts.values, edgecolor='black', alpha=0.7, color='plum')\n", "ax4.set_xticks(range(len(source_counts)))\n", "ax4.set_xticklabels(source_counts.index, rotation=45, ha='right')\n", "ax4.set_xlabel('Data Source')\n", "ax4.set_ylabel('Frequency')\n", "ax4.set_title(f'Distribution of Data Source\\n(n={len(data_source):,})')\n", "ax4.grid(True, alpha=0.3, axis='y')\n", "\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "# Print summary statistics\n", "print(\"=\" * 60)\n", "print(\"SUMMARY STATISTICS\")\n", "print(\"=\" * 60)\n", "print(f\"\\nTotal number of songs: {len(all_mashups):,}\")\n", "\n", "# Calculate total hours\n", "total_duration_seconds = all_mashups['duration'].sum()\n", "total_hours = total_duration_seconds / 3600\n", "print(f\"Total duration: {total_duration_seconds:,.2f} seconds\")\n", "print(f\"Total duration: {total_hours:,.2f} hours\")\n", "print(f\"Total duration: {total_hours/24:,.2f} days\")\n", "\n", "print(\"\\n\" + \"=\" * 60)\n", "print(\"FILE AVAILABILITY STATISTICS\")\n", "print(\"=\" * 60)\n", "\n", "# Rows where s3_filepath is not None or output_id is not None\n", "has_s3_or_output = all_mashups['s3_filepath'].notna() | all_mashups['output_id'].notna()\n", "num_has_s3_or_output = has_s3_or_output.sum()\n", "pct_has_s3_or_output = (num_has_s3_or_output / len(all_mashups)) * 100\n", "print(f\"\\nRows with s3_filepath OR output_id (not None):\")\n", "print(f\" Count: {num_has_s3_or_output:,}\")\n", "print(f\" Percentage: {pct_has_s3_or_output:.2f}%\")\n", "\n", "# Rows where source_urls is not None\n", "has_source_urls = all_mashups['source_urls'].notna()\n", "num_has_source_urls = has_source_urls.sum()\n", "pct_has_source_urls = (num_has_source_urls / len(all_mashups)) * 100\n", "print(f\"\\nRows with source_urls (not None):\")\n", "print(f\" Count: {num_has_source_urls:,}\")\n", "print(f\" Percentage: {pct_has_source_urls:.2f}%\")\n", "\n", "print(\"\\n\" + \"=\" * 60)\n", "print(\"DETAILED STATISTICS BY FIELD\")\n", "print(\"=\" * 60)\n", "\n", "print(f\"\\nRelease Date:\")\n", "print(f\" Non-null values: {all_mashups['release_date'].notna().sum():,}\")\n", "print(f\" Null values: {all_mashups['release_date'].isna().sum():,}\")\n", "print(f\" Zero values (excluded): {(all_mashups['release_date'] == 0).sum():,}\")\n", "if len(release_dates_filtered) > 0:\n", " print(f\" Min (filtered): {int(release_dates_filtered.min())}\")\n", " print(f\" Max (filtered): {int(release_dates_filtered.max())}\")\n", " print(f\" Mean: {release_dates_filtered.mean():.0f}\")\n", " print(f\" Median: {release_dates_filtered.median():.0f}\")\n", " print(f\" Std Dev: {release_dates_filtered.std():.0f}\")\n", "\n", "print(f\"\\nDuration:\")\n", "print(f\" Non-null values: {all_mashups['duration'].notna().sum():,}\")\n", "print(f\" Null values: {all_mashups['duration'].isna().sum():,}\")\n", "if len(durations) > 0:\n", " print(f\" Min: {durations.min():.2f} seconds\")\n", " print(f\" Max: {durations.max():.2f} seconds\")\n", " print(f\" Mean: {durations.mean():.2f} seconds\")\n", " print(f\" Median: {durations.median():.2f} seconds\")\n", " print(f\" Std Dev: {durations.std():.2f} seconds\")\n", " print(f\" 95th percentile: {durations.quantile(0.95):.2f} seconds\")\n", "\n", "print(f\"\\nMashup Strength:\")\n", "print(strength_counts)\n", "\n", "print(f\"\\nData Source:\")\n", "print(source_counts)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "35", "metadata": {}, "outputs": [], "source": [ "sfx_metas = read_jsonl(\"/app2/suno/data/sara/sfx_get_beats/combined_v3_w_extreme_metas_v0.jsonl\")\n", "sfx_metas[0].keys()" ] }, { "cell_type": "code", "execution_count": null, "id": "36", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text import read_jsonl, write_jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "37", "metadata": {}, "outputs": [], "source": [ "for m in sfx_metas:\n", " id = m['id']\n", " m['s3_filepath'] = f\"s3://suno-data/datasets/bundles/v5/sfx_all_processed/opus_audio_truncated/{id}.opus\"" ] }, { "cell_type": "code", "execution_count": null, "id": "38", "metadata": {}, "outputs": [], "source": [ "write_jsonl(sfx_metas, \"/app2/suno/data/sara/sfx_get_beats/combined_v3_w_extreme_metas_v0_truncated.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "39", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_clean", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }