{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import json\n", "import os\n", "from suno_utils.utils.text import read_jsonl" ] }, { "cell_type": "code", "execution_count": 209, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "274275\n" ] } ], "source": [ "# now we load the original metas to get the artists and song titles\n", "METAS_DIR = \"/app/suno/tmp\"\n", "#raw_metas = read_jsonl(os.path.join(METAS_DIR, \"raw_discogs_subset_metas.jsonl\"))\n", "#raw_metas = read_jsonl(os.path.join(METAS_DIR, \"raw_discogs_metas.jsonl\"))\n", "#raw_metas = read_jsonl(os.path.join(METAS_DIR, \"raw_genius_metas.jsonl\"))\n", "raw_metas = read_jsonl(os.path.join(METAS_DIR, \"raw_imslp_metas.jsonl\"))\n", "\n", "#raw_metas = read_jsonl(\"/app/suno/tmp/raw_deezer_metas.jsonl\")\n", "#raw_metas = read_jsonl(\"/app/suno/tmp/raw_youtube_music_metas.jsonl\")\n", "#raw_metas = read_jsonl(\"/app/suno/tmp/raw_pond5_metas.jsonl\")\n", "\n", "print(len(raw_metas))\n", "#dataset_name = \"imslp\"" ] }, { "cell_type": "code", "execution_count": 211, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'id': 'a6cfab9a-817d-4b93-a446-76a500dd2e87', 's3_filepath': 's3://suno-data/datasets/harvest/imslp/audio/a6cfab9a-817d-4b93-a446-76a500dd2e87.mp3', 'duration_s': 153, 'composer': 'Fauré, Gabriel', 'recording_category': 'Commercial Recordings', 'tags': ['Romantic', 'voice', 'Fauré', 'piano', 'Gabriel'], 'tags_redacted': ['Romantic', 'piano', 'voice'], 'instruments': 'voice, piano', 'genre': 'Romantic', 'original_id': '54698'}\n" ] } ], "source": [ "# find a meta with empty title\n", "for meta in raw_metas:\n", " if meta.get(\"title\", \"\") == \"\":\n", " print(meta)\n", " break\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "titles = []\n", "for meta in raw_metas:\n", " title = meta.get(\"title\", \"\")\n", " artists = meta.get(\"artists\", [])\n", " artist_string = \", \".join(artist[\"name\"] for artist in artists)\n", " combined_title = f\"{title} - {artist_string}\"\n", " titles.append({\n", " \"id\" : meta[\"id\"],\n", " \"title\": combined_title\n", "\n", " })" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "titles = []\n", "for meta in raw_metas:\n", " genius_slug = meta.get(\"genius_slug\", \"\")\n", " # split on - \n", " combined_title = genius_slug.replace(\"-\", \" \") \n", " combined_title = combined_title.replace(\"lyrics\", \" \")\n", " combined_title = combined_title.strip()\n", " #combined_title = f\"{title} - {artist_string}\"\n", " titles.append({\n", " \"id\" : meta[\"id\"],\n", " \"title\": combined_title\n", "\n", " })" ] }, { "cell_type": "code", "execution_count": 212, "metadata": {}, "outputs": [], "source": [ "titles = []\n", "for meta in raw_metas:\n", " title = meta.get(\"title\", \"\")\n", " composer = meta.get(\"composer\", \"\")\n", "\n", " # check for empty title\n", " if title == \"\":\n", " tags = meta.get(\"tags\", [])\n", " if len(tags) < 1:\n", " continue\n", " # create title from tags\n", " title = \" \".join(tags)\n", "\n", " combined_title = f\"{title} - {composer}\"\n", " #combined_title = f\"{title} - {artist_string}\"\n", " titles.append({\n", " \"id\" : meta[\"id\"],\n", " \"title\": combined_title\n", "\n", " })" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "titles = []\n", "for meta in raw_metas:\n", " title = meta.get(\"title\", \"\")\n", " artist = meta.get(\"artists\", \"\")\n", " # split on - \n", " combined_title = f\"{title} - {artist}\"\n", " #combined_title = f\"{title} - {artist_string}\"\n", " titles.append({\n", " \"id\" : meta[\"id\"],\n", " \"title\": combined_title\n", "\n", " })" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [], "source": [ "titles = []\n", "for meta in raw_metas:\n", " title = meta.get(\"id\", \"\")\n", " combined_title = title.split(\"-\")[1:]\n", " combined_title = \" \".join(combined_title)\n", "\n", " #artist = meta.get(\"artists\", \"\")\n", " # split on - \n", " #combined_title = f\"{title} - {artist}\"\n", " #combined_title = f\"{title} - {artist_string}\"\n", " titles.append({\n", " \"id\" : meta[\"id\"],\n", " \"title\": combined_title\n", "\n", " })" ] }, { "cell_type": "code", "execution_count": 213, "metadata": {}, "outputs": [], "source": [ "# save out json with the titles\n", "with open(\"/home/christian/code/christian/metadata/dedup/imslp_titles.json\", \"w\") as f:\n", " json.dump(titles, f)\n" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "id2title = {d[\"id\"]: d[\"title\"] for d in titles}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for title in titles[20:30]:\n", " print(title)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# now comnbine all titles into single json \n", "\n", "title_filepaths = [\n", " \"/home/christian/code/christian/metadata/dedup/imslp_titles.json\",\n", " \"/home/christian/code/christian/metadata/dedup/deezer_titles.json\",\n", " \"/home/christian/code/christian/metadata/dedup/youtube_music_titles.json\",\n", " \"/home/christian/code/christian/metadata/dedup/pond5_titles.json\",\n", " \"/home/christian/code/christian/metadata/dedup/discogs_titles.json\",\n", " \"/home/christian/code/christian/metadata/dedup/genius_titles.json\",\n", " \"/home/christian/code/christian/metadata/dedup/discogs_subset_titles.json\",\n", "]\n", "\n", "all_titles = []\n", "for title_filepath in title_filepaths:\n", " with open(title_filepath, \"r\") as f:\n", " titles = json.load(f)\n", " all_titles.extend(titles)\n", "\n", "print(len(all_titles))\n", "with open(\"/home/christian/code/christian/metadata/dedup/combined_titles.json\", \"w\") as f:\n", " json.dump(all_titles, f)\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "#filepath = \"/home/christian/code/christian/metadata/dedup/combined_title_clusters_jac-0.92_lsh-0.85.json\"\n", "#filepath = \"/home/christian/code/christian/metadata/dedup/combined_title_clusters_jac-0.80_lsh-0.60.json\"\n", "#filepath = \"/home/christian/code/christian/metadata/dedup/combined_title_clusters.json\"\n", "filepath = \"/home/christian/code/christian/metadata/dedup/youtube_music_title_clusters_jac-0.80.json\"\n", "\n", "with open(filepath, \"r\") as fp:\n", " results = json.load(fp)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'size': 3, 'items': [{'id': '56295447-ec29-4a0f-b800-90acef1af527', 'title': 'Contrasts, for Clarinet, Violin and Piano, Sz. 111 - Bartók, Béla'}, {'id': '5abc5a01-f5ed-4c5d-a754-a173923edc52', 'title': 'Contrasts, for Clarinet, Violin and Piano, Sz. 111 - Bartók, Béla'}, {'id': '5d106358-fa53-4974-9c9f-79a528deff40', 'title': 'Contrasts, for Clarinet, Violin and Piano, Sz. 111 - Bartók, Béla'}]}\n" ] } ], "source": [ "print(results[0])" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "601,752\n" ] } ], "source": [ "# count the total number of duplicates\n", "total_duplicates = [result[\"size\"] for result in results]\n", "print(f\"{sum(total_duplicates):,}\")" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "# convert to jsonl \n", "import random\n", "results_list = []\n", "\n", "# sort by size\n", "results_list = sorted(results, key=lambda x: x['size'], reverse=True)\n", "\n", "results_list = results_list[:10000]\n", "#results_list = random.sample(results_list, 10000)\n", "\n", "results_dict = {}\n", "for idx, result in enumerate(results_list):\n", " results_dict[idx] = result\n", "\n", "with open(\"/home/christian/code/christian/metadata/dedup/youtube_music_title_clusters_jac-0.80_lsh-0.60_10k.json\", \"w\") as f:\n", " json.dump(results_dict, f, indent=2)\n", "\n" ] }, { "cell_type": "code", "execution_count": 165, "metadata": {}, "outputs": [], "source": [ "# Sort clusters by size in descending order\n", "sorted_results = sorted(results, key=lambda x: x['size'], reverse=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "results_short_ids = [result for result in results if len(result[\"items\"][0][\"id\"]) < 16]\n", "print(len(results_short_ids))" ] }, { "cell_type": "code", "execution_count": 86, "metadata": {}, "outputs": [], "source": [ "# Sort clusters by size in descending order\n", "sorted_results_short_ids = sorted(results_short_ids, key=lambda x: x['size'], reverse=True)" ] }, { "cell_type": "code", "execution_count": 167, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5052004\n" ] } ], "source": [ "# sizes \n", "sizes = [result[\"size\"] for result in results]\n", "print(len(sizes))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(sizes[0])" ] }, { "cell_type": "code", "execution_count": 168, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGjCAYAAAD3mbWOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAApzklEQVR4nO3df3Ac5X3H8Y8kW2c7tmQL4ZNlJAsHcCpsS0U/DqWY4PomQlATTNtxUyYI0RGTVGTSUSCRkikuHYo8detRjTdxJxlH0zaNDR0smBhoiLAQpALLMsIYBYOoDApwZ4zjO0uAjE9P/6A+OCTZPumk29W+XzM7k9t97tnvPj5xn+w+u5dijDECAACwidRkFwAAAPBZhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGAr0x5Ojhw5ouLi4ugyd+5ctba2TncZAADAplKS+cN/g4ODKigo0JtvvqkvfOELySoDAADYSFIv6zz22GNat24dwQQAAETNivcNHR0d2rJli7q7u/Xuu+9qz549uvnmm2PaWJalLVu2KBAIqKioSA8++KDKy8tH9fXQQw/ptttui2v/IyMjeuedd7RgwQKlpKTEWz4AAEgCY4xOnTql3Nxcpaae59yIidPjjz9ufvjDH5pHHnnESDJ79uyJ2b5r1y6Tnp5udu7caV555RVTW1trFi5caILBYEy7UChkLr74YvPhhx/Gtf+BgQEjiYWFhYWFhcWBy8DAwHm/6yc15yQlJWXUmROfz6eysjJt375d0idnOvLy8vTtb39bDQ0N0Xb//u//rv/+7//Wf/zHf5xzH8PDwxoeHo6+DoVCys/P18DAgDIyMiZaOgAAmEbhcFh5eXk6efKkMjMzz9k27ss653L69Gl1d3ersbExui41NVV+v1+dnZ0xbR966CHdeeed5+2zqalJ991336j1GRkZhBMAABzmQqZkJHRC7PHjxxWJROT1emPWe71eBQKB6OtQKKT9+/ersrLyvH02NjYqFApFl4GBgUSWDAAAbCahZ04uVGZmpoLB4AW19Xg88ng8sixLlmUpEolMcXUAACCZEnrmJDs7W2lpaaOCRzAYVE5OzqT6rqurU29vr7q6uibVDwAAsLeEhpP09HSVlJSora0tum5kZERtbW2qqKiYVN+WZamwsFBlZWWTLRMAANhY3Jd1BgcH1dfXF33d39+vnp4eZWVlKT8/X/X19aqurlZpaanKy8vV3NysoaEh1dTUTKrQuro61dXVKRwOn3eWLwAAcK64w8mBAwe0du3a6Ov6+npJUnV1tVpaWrRx40a99957uvfeexUIBFRcXKwnn3xy1CRZAACAsST1t3Xi8dkJsa+99ppCoRC3EgMA4BBnr3xcyPe3Y8LJWfEcHAAAsId4vr+T+sN/AAAAn0c4AQAAtuKYcMKtxAAAuANzTgAAwJRjzgkAAHCspPy2jp0VNOyNeX10841JqgQAAHdyzJkT5pwAAOAOjgkn/PAfAADu4JhwAgAA3IFwAgAAbIVwAgAAbMUx4YQJsQAAuINjwgkTYgEAcAfHhBMAAOAOhBMAAGArhBMAAGArhBMAAGArjgkn3K0DAIA7OCaccLcOAADu4JhwAgAA3IFwAgAAbIVwAgAAbIVwAgAAbIVwAgAAbIVwAgAAbIVwAgAAbMUx4YSHsAEA4A6OCSc8hA0AAHdwTDgBAADuQDgBAAC2QjgBAAC2QjgBAAC2QjgBAAC2QjgBAAC2QjgBAAC2kpRw0t/fr7Vr16qwsFCrVq3S0NBQMsoAAAA2NCsZO7399tt1//33a82aNTpx4oQ8Hk8yygAAADY07eHklVde0ezZs7VmzRpJUlZW1nSXAAAAbCzuyzodHR1av369cnNzlZKSotbW1lFtLMtSQUGB5syZI5/Pp/3790e3vf7665o/f77Wr1+vq666Sg888MCkDgAAAMwscYeToaEhFRUVybKsMbfv3r1b9fX12rRpkw4ePKiioiJVVlbq2LFjkqQzZ87o2Wef1Y9+9CN1dnbqqaee0lNPPTW5owAAADNG3OGkqqpK999/vzZs2DDm9q1bt6q2tlY1NTUqLCzUjh07NG/ePO3cuVOStHTpUpWWliovL08ej0c33HCDenp6xt3f8PCwwuFwzAIAAGauhN6tc/r0aXV3d8vv93+6g9RU+f1+dXZ2SpLKysp07Ngx/f73v9fIyIg6Ojr0B3/wB+P22dTUpMzMzOiSl5eXyJIBAIDNJDScHD9+XJFIRF6vN2a91+tVIBCQJM2aNUsPPPCArr32Wq1evVqXX365/uRP/mTcPhsbGxUKhaLLwMBAIksGAAA2k5RbiauqqlRVVXVBbT0ejzwejyzLkmVZikQiU1wdAABIpoSeOcnOzlZaWpqCwWDM+mAwqJycnEn1XVdXp97eXnV1dU2qHwAAYG8JDSfp6ekqKSlRW1tbdN3IyIja2tpUUVGRyF0BAIAZKu7LOoODg+rr64u+7u/vV09Pj7KyspSfn6/6+npVV1ertLRU5eXlam5u1tDQkGpqaiZVKJd1AABwhxRjjInnDe3t7Vq7du2o9dXV1WppaZEkbd++XVu2bFEgEFBxcbG2bdsmn8+XkILD4bAyMzMVCoWUkZGRkD4/q6Bhb8zro5tvTPg+AABwm3i+v+MOJ8ny2TMnr732GuEEAAAHiSecJOVXiSeCCbEAALiDY8IJAABwB8IJAACwFceEE8uyVFhYqLKysmSXAgAAppBjwglzTgAAcAfHhBMAAOAOhBMAAGArjgknzDkBAMAdHBNOmHMCAIA7OCacAAAAdyCcAAAAWyGcAAAAW3FMOGFCLAAA7uCYcMKEWAAA3MEx4QQAALgD4QQAANgK4QQAANgK4QQAANiKY8IJd+sAAOAOjgkn3K0DAIA7OCacAAAAdyCcAAAAWyGcAAAAWyGcAAAAWyGcAAAAWyGcAAAAWyGcAAAAW3FMOOEhbAAAuINjwgkPYQMAwB0cE04AAIA7EE4AAICtEE4AAICtEE4AAICtEE4AAICtEE4AAICtEE4AAICtzErGTgsKCpSRkaHU1FQtWrRI+/btS0YZAADAhpISTiTpf/7nfzR//vxk7R4AANgUl3UAAICtxB1OOjo6tH79euXm5iolJUWtra2j2liWpYKCAs2ZM0c+n0/79++P2Z6SkqKvfOUrKisr089//vMJFw8AAGaeuMPJ0NCQioqKZFnWmNt3796t+vp6bdq0SQcPHlRRUZEqKyt17NixaJvnnntO3d3deuyxx/TAAw/o0KFDEz8CAAAwo8QdTqqqqnT//fdrw4YNY27funWramtrVVNTo8LCQu3YsUPz5s3Tzp07o22WLl0qSVqyZIluuOEGHTx4cNz9DQ8PKxwOxywAAGDmSuick9OnT6u7u1t+v//THaSmyu/3q7OzU9InZ15OnTolSRocHNTTTz+tK6+8ctw+m5qalJmZGV3y8vISWTIAALCZhIaT48ePKxKJyOv1xqz3er0KBAKSpGAwqGuuuUZFRUW6+uqrddttt6msrGzcPhsbGxUKhaLLwMBAIksGAAA2M+23Ei9fvlwvvfTSBbf3eDzyeDyyLEuWZSkSiUxhdQAAINkSeuYkOztbaWlpCgaDMeuDwaBycnIm1XddXZ16e3vV1dU1qX4AAIC9JTScpKenq6SkRG1tbdF1IyMjamtrU0VFRSJ3BQAAZqi4L+sMDg6qr68v+rq/v189PT3KyspSfn6+6uvrVV1drdLSUpWXl6u5uVlDQ0OqqamZVKFc1gEAwB1SjDEmnje0t7dr7dq1o9ZXV1erpaVFkrR9+3Zt2bJFgUBAxcXF2rZtm3w+X0IKDofDyszMVCgUUkZGRkL6/KyChr0xr49uvjHh+wAAwG3i+f6OO5wkG+EEAADnief72zG/rWNZlgoLC8952zEAAHA+x4QT7tYBAMAdHBNOAACAOzgmnHBZBwAAd3BMOOGyDgAA7uCYcAIAANyBcAIAAGzFMeGEOScAALiDY8IJc04AAHAHx4QTAADgDoQTAABgK4QTAABgK44JJ0yIBQDAHRwTTpgQCwCAOzgmnAAAAHcgnAAAAFshnAAAAFshnAAAAFtxTDjhbh0AANzBMeGEu3UAAHAHx4QTAADgDoQTAABgK4QTAABgK4QTAABgK4QTAABgK4QTAABgK4QTAABgK44JJzyEDQAAd3BMOOEhbAAAuINjwgkAAHAHwgkAALAVwgkAALAVwgkAALAVwgkAALAVwgkAALAVwgkAALCVpIWTDz74QMuWLdPdd9+drBIAAIANJS2c/MM//IOuvvrqZO0eAADYVFLCyeuvv65XX31VVVVVydg9AACwsbjDSUdHh9avX6/c3FylpKSotbV1VBvLslRQUKA5c+bI5/Np//79MdvvvvtuNTU1TbhoAAAwc8UdToaGhlRUVCTLssbcvnv3btXX12vTpk06ePCgioqKVFlZqWPHjkmSHn30UV1xxRW64oorJlc5AACYkWbF+4aqqqpzXo7ZunWramtrVVNTI0nasWOH9u7dq507d6qhoUHPP/+8du3apYcffliDg4P6+OOPlZGRoXvvvXfM/oaHhzU8PBx9HQ6H4y0ZAAA4SELnnJw+fVrd3d3y+/2f7iA1VX6/X52dnZKkpqYmDQwM6OjRo/qnf/on1dbWjhtMzrbPzMyMLnl5eYksGQAA2ExCw8nx48cViUTk9Xpj1nu9XgUCgQn12djYqFAoFF0GBgYSUSoAALCpuC/rJNLtt99+3jYej0cej0eWZcmyLEUikakvDAAAJE1Cz5xkZ2crLS1NwWAwZn0wGFROTs6k+q6rq1Nvb6+6urom1Q8AALC3hIaT9PR0lZSUqK2tLbpuZGREbW1tqqioSOSuAADADBX3ZZ3BwUH19fVFX/f396unp0dZWVnKz89XfX29qqurVVpaqvLycjU3N2toaCh6985EcVkHAAB3SDHGmHje0N7errVr145aX11drZaWFknS9u3btWXLFgUCARUXF2vbtm3y+XwJKTgcDiszM1OhUEgZGRkJ6fOzChr2xrw+uvnGhO8DAAC3ief7O+5wkmyEEwAAnCee7++k3q0Tj2Rd1vl8WJEILAAATKWk/SpxvLhbBwAAd3BMOAEAAO7gmHBiWZYKCwtVVlaW7FIAAMAUckw44bIOAADu4JhwAgAA3IFwAgAAbMUx4YQ5JwAAuINjwglzTgAAcAfHhBMAAOAOhBMAAGArhBMAAGArjgknTIgFAMAdHBNOmBALAIA7OCacAAAAdyCcAAAAWyGcAAAAWyGcAAAAW3FMOOFuHQAA3MEx4YS7dQAAcAfHhBMAAOAOhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGArhBMAAGArjgknPIQNAAB3cEw44SFsAAC4g2PCCQAAcAfCCQAAsBXCCQAAsBXCCQAAsJVZyS7AiQoa9sa8Prr5xiRVAgDAzMOZEwAAYCuEEwAAYCvTHk5Onjyp0tJSFRcXa+XKlfrJT34y3SUAAAAbm/Y5JwsWLFBHR4fmzZunoaEhrVy5Urfccosuuuii6S4FAADY0LSfOUlLS9O8efMkScPDwzLGyBgz3WUAAACbijucdHR0aP369crNzVVKSopaW1tHtbEsSwUFBZozZ458Pp/2798fs/3kyZMqKirSJZdconvuuUfZ2dkTPgAAADCzxB1OhoaGVFRUJMuyxty+e/du1dfXa9OmTTp48KCKiopUWVmpY8eORdssXLhQL730kvr7+/Wf//mfCgaDEz8CAAAwo8QdTqqqqnT//fdrw4YNY27funWramtrVVNTo8LCQu3YsUPz5s3Tzp07R7X1er0qKirSs88+O+7+hoeHFQ6HYxYAADBzJXTOyenTp9Xd3S2/3//pDlJT5ff71dnZKUkKBoM6deqUJCkUCqmjo0MrVqwYt8+mpiZlZmZGl7y8vESWDAAAbCah4eT48eOKRCLyer0x671erwKBgCTpzTff1Jo1a1RUVKQ1a9bo29/+tlatWjVun42NjQqFQtFlYGAgkSUDAACbmfZbicvLy9XT03PB7T0ejzwejyzLkmVZikQiU1ccAABIuoSeOcnOzlZaWtqoCa7BYFA5OTmT6ruurk69vb3q6uqaVD8AAMDeEhpO0tPTVVJSora2tui6kZERtbW1qaKiIpG7AgAAM1Tcl3UGBwfV19cXfd3f36+enh5lZWUpPz9f9fX1qq6uVmlpqcrLy9Xc3KyhoSHV1NRMqlAu6wAA4A4pJs7Hs7a3t2vt2rWj1ldXV6ulpUWStH37dm3ZskWBQEDFxcXatm2bfD5fQgoOh8PKzMxUKBRSRkZGQvr8rIKGvXG/5+jmGxNeBwAAM0k8399xh5NkI5wAAOA88Xx/T/tv60yUZVkqLCxUWVlZsksBAABTyDHhhLt1AABwB8eEEwAA4A6OCSdc1gEAwB0cE064rAMAgDtM++PrZ6Kx7vDhDh4AACbGMWdOAACAOzgmnDDnBAAAd3BMOGHOCQAA7uCYcAIAANyBcAIAAGyFcAIAAGzFMeGECbEAALiDY8IJE2IBAHAHx4QTAADgDoQTAABgK4QTAABgK4QTAABgK44JJ9ytAwCAOzgmnHC3DgAA7uCYcAIAANyBcAIAAGyFcAIAAGyFcAIAAGyFcAIAAGyFcAIAAGyFcAIAAGxlVrILuFCWZcmyLEUikWSXckEKGvbGvD66+cYkVQIAgLM45swJD2EDAMAdHBNOAACAOxBOAACArRBOAACArRBOAACArRBOAACArRBOAACArRBOAACArUx7OBkYGNB1112nwsJCrV69Wg8//PB0lwAAAGxs2p8QO2vWLDU3N6u4uFiBQEAlJSW64YYb9IUvfGG6SwEAADY07eFkyZIlWrJkiSQpJydH2dnZOnHiBOEEAABImsBlnY6ODq1fv165ublKSUlRa2vrqDaWZamgoEBz5syRz+fT/v37x+yru7tbkUhEeXl5cRcOAABmprjPnAwNDamoqEh33HGHbrnlllHbd+/erfr6eu3YsUM+n0/Nzc2qrKzUkSNHtHjx4mi7EydO6LbbbtNPfvKTyR2BQ3z+hwAlfgwQAICxxB1OqqqqVFVVNe72rVu3qra2VjU1NZKkHTt2aO/evdq5c6caGhokScPDw7r55pvV0NCgL3/5y+fc3/DwsIaHh6Ovw+FwvCUDAAAHSejdOqdPn1Z3d7f8fv+nO0hNld/vV2dnpyTJGKPbb79df/zHf6xvfOMb5+2zqalJmZmZ0YVLQAAAzGwJDSfHjx9XJBKR1+uNWe/1ehUIBCRJv/nNb7R79261traquLhYxcXFevnll8fts7GxUaFQKLoMDAwksmQAAGAz0363zjXXXKORkZELbu/xeOTxeKawIgAAYCcJPXOSnZ2ttLQ0BYPBmPXBYFA5OTmT6tuyLBUWFqqsrGxS/QAAAHtLaDhJT09XSUmJ2traoutGRkbU1tamioqKSfVdV1en3t5edXV1TbZMAABgY3Ff1hkcHFRfX1/0dX9/v3p6epSVlaX8/HzV19erurpapaWlKi8vV3Nzs4aGhqJ370yUZVmyLEuRSGRS/QAAAHtLMcaYeN7Q3t6utWvXjlpfXV2tlpYWSdL27du1ZcsWBQIBFRcXa9u2bfL5fAkpOBwOKzMzU6FQSBkZGQnp87PGeh7JdOG5JwCAmSqe7++4w0myEU4AAHCeeL6/p/1XiSeKCbEAALiDY8IJE2IBAHAHx4QTAADgDo4JJ1zWAQDAHRwTTrisAwCAOzgmnAAAAHcgnAAAAFtxTDhhzgkAAO7gmHDCnBMAANwh7t/WwdQZ6+m0PDUWAOA2jjlzAgAA3IFwAgAAbMUx4YQJsQAAuINjwgkTYgEAcAfHhBMAAOAOhBMAAGArhBMAAGArhBMAAGArjgkn3K0DAIA7OOYJsXV1daqrq1M4HFZmZmayy0kaniILAJjpHHPmBAAAuAPhBAAA2ArhBAAA2ArhBAAA2ArhBAAA2ArhBAAA2ArhBAAA2IpjnnNiWZYsy1IkEkl2KdNqrOeaAAAwkznmzEldXZ16e3vV1dWV7FIAAMAUckw4AQAA7kA4AQAAtkI4AQAAtkI4AQAAtkI4AQAAtkI4AQAAtuKY55zgwo31bJSjm29MQiUAAMQvKWdONmzYoEWLFunP/uzPkrF7AABgY0kJJ9/5znf0b//2b8nYNQAAsLmkhJPrrrtOCxYsSMauAQCAzcUdTjo6OrR+/Xrl5uYqJSVFra2to9pYlqWCggLNmTNHPp9P+/fvT0StAADABeKeEDs0NKSioiLdcccduuWWW0Zt3717t+rr67Vjxw75fD41NzersrJSR44c0eLFixNSNGJdyI8Dfr4NE2QBAHYVdzipqqpSVVXVuNu3bt2q2tpa1dTUSJJ27NihvXv3aufOnWpoaIi7wOHhYQ0PD0dfh8PhuPsAAADOkdA5J6dPn1Z3d7f8fv+nO0hNld/vV2dn54T6bGpqUmZmZnTJy8tLVLkAAMCGEhpOjh8/rkgkIq/XG7Pe6/UqEAhEX/v9fv35n/+5Hn/8cV1yySXnDC6NjY0KhULRZWBgIJElAwAAm0nKQ9h+/etfX3Bbj8cjj8czhdUAAAA7SWg4yc7OVlpamoLBYMz6YDConJycSfVtWZYsy1IkEplUP/gET5EFANhVQi/rpKenq6SkRG1tbdF1IyMjamtrU0VFxaT6rqurU29vr7q6uiZbJgAAsLG4z5wMDg6qr68v+rq/v189PT3KyspSfn6+6uvrVV1drdLSUpWXl6u5uVlDQ0PRu3cmijMnU4/bjQEAdpBijDHxvKG9vV1r164dtb66ulotLS2SpO3bt2vLli0KBAIqLi7Wtm3b5PP5ElJwOBxWZmamQqGQMjIyEtLnZ13IM0PcgnACAEiUeL6/4w4nyUY4mT6EEwBAosTz/Z2U39aZCMuyVFhYqLKysmSXAgAAppBjwgkTYgEAcAfHhBMAAOAOSXkI20Rwt449cEcPAGCqOebMCZd1AABwB8eEEwAA4A6EEwAAYCvMOcG4EvXMF37HBwAQD8ecOWHOCQAA7uCYcAIAANyBcAIAAGyFcAIAAGyFCbGYFCa7AgASzTFnTpgQCwCAOzgmnAAAAHcgnAAAAFshnAAAAFshnAAAAFshnAAAAFvhVmI4BrctA4A7OObMCbcSAwDgDo4JJwAAwB0IJwAAwFYIJwAAwFYIJwAAwFYIJwAAwFYIJwAAwFZ4zgkSbqznkZyvzVjPK7mQfiZSjxOejeLEmgEgURxz5oTnnAAA4A6OCScAAMAdCCcAAMBWCCcAAMBWCCcAAMBWCCcAAMBWCCcAAMBWCCcAAMBWCCcAAMBWkhJOfvnLX2rFihW6/PLL9dOf/jQZJQAAAJua9sfXnzlzRvX19dq3b58yMzNVUlKiDRs26KKLLpruUgAAgA1N+5mT/fv368orr9TSpUs1f/58VVVV6Ve/+tV0lwEAAGwq7nDS0dGh9evXKzc3VykpKWptbR3VxrIsFRQUaM6cOfL5fNq/f3902zvvvKOlS5dGXy9dulRvv/32xKoHAAAzTtzhZGhoSEVFRbIsa8ztu3fvVn19vTZt2qSDBw+qqKhIlZWVOnbs2IQKHB4eVjgcjlkAAMDMFfeck6qqKlVVVY27fevWraqtrVVNTY0kaceOHdq7d6927typhoYG5ebmxpwpefvtt1VeXj5uf01NTbrvvvviLRMOU9Cwd9reN9Z7jm6+cUL7n8i+7GY6x2Mqff44xjqGC2mTqH3Z3Uz5d0+UmfBvOlF2/CwkdM7J6dOn1d3dLb/f/+kOUlPl9/vV2dkpSSovL9fhw4f19ttva3BwUE888YQqKyvH7bOxsVGhUCi6DAwMJLJkAABgMwm9W+f48eOKRCLyer0x671er1599dVPdjhrlv75n/9Za9eu1cjIiL73ve+d804dj8cjj8eTyDIBAICNTfutxJJ000036aabborrPZZlybIsRSKRKaoKAADYQUIv62RnZystLU3BYDBmfTAYVE5OzqT6rqurU29vr7q6uibVDwAAsLeEhpP09HSVlJSora0tum5kZERtbW2qqKiYVN+WZamwsFBlZWWTLRMAANhY3Jd1BgcH1dfXF33d39+vnp4eZWVlKT8/X/X19aqurlZpaanKy8vV3NysoaGh6N07E1VXV6e6ujqFw2FlZmZOqi8AAGBfcYeTAwcOaO3atdHX9fX1kqTq6mq1tLRo48aNeu+993TvvfcqEAiouLhYTz755KhJsgAAAGOJO5xcd911Msacs81dd92lu+66a8JFjYUJsQAAuENSfpV4IpgQCwCAOzgmnAAAAHcgnAAAAFtxTDjhVmIAANzBMeGEOScAALiDY8IJAABwB8IJAACwlaT88N9EnH3OyZkzZyRJ4XB4SvYzMvzBlPQLe7P752mq6pPGrnEq9zdVPn8cYx3DhbRJ1L7sbqb8uyfKTPg3najp+iyc7fN8z0qTpBRzIa1s5He/+53y8vKSXQYAAJiAgYEBXXLJJeds47hwMjIyonfeeUcLFixQSkpKwvoNh8PKy8vTwMCAMjIyEtbvTMDYjI1xGR9jMz7GZmyMy/hmytgYY3Tq1Cnl5uYqNfXcs0occ1nnrNTU1PMmrsnIyMhw9D/+VGJsxsa4jI+xGR9jMzbGZXwzYWwu9Id7mRALAABshXACAABshXDy/zwejzZt2iSPx5PsUmyHsRkb4zI+xmZ8jM3YGJfxuXFsHDchFgAAzGycOQEAALZCOAEAALZCOAEAALZCOAEAALZCOPl/lmWpoKBAc+bMkc/n0/79+5Nd0pT6u7/7O6WkpMQsX/rSl6LbP/roI9XV1emiiy7S/Pnz9ad/+qcKBoMxfbz11lu68cYbNW/ePC1evFj33HNP9LePnKKjo0Pr169Xbm6uUlJS1NraGrPdGKN7771XS5Ys0dy5c+X3+/X666/HtDlx4oRuvfVWZWRkaOHChfqrv/orDQ4OxrQ5dOiQ1qxZozlz5igvL0//+I//ONWHNmnnG5vbb7991Gfo+uuvj2kzE8emqalJZWVlWrBggRYvXqybb75ZR44ciWmTqL+f9vZ2XXXVVfJ4PLrsssvU0tIy1Yc3KRcyNtddd92oz803v/nNmDYzbWx+/OMfa/Xq1dGHqFVUVOiJJ56Ibnfr5+WcDMyuXbtMenq62blzp3nllVdMbW2tWbhwoQkGg8kubcps2rTJXHnllebdd9+NLu+99150+ze/+U2Tl5dn2trazIEDB8zVV19tvvzlL0e3nzlzxqxcudL4/X7z4osvmscff9xkZ2ebxsbGZBzOhD3++OPmhz/8oXnkkUeMJLNnz56Y7Zs3bzaZmZmmtbXVvPTSS+amm24yl156qfnwww+jba6//npTVFRknn/+efPss8+ayy67zHz961+Pbg+FQsbr9Zpbb73VHD582PziF78wc+fONf/6r/86XYc5Iecbm+rqanP99dfHfIZOnDgR02Ymjk1lZaX52c9+Zg4fPmx6enrMDTfcYPLz883g4GC0TSL+fv73f//XzJs3z9TX15ve3l7z4IMPmrS0NPPkk09O6/HG40LG5itf+Yqpra2N+dyEQqHo9pk4No899pjZu3evee2118yRI0fMD37wAzN79mxz+PBhY4x7Py/nQjgxxpSXl5u6urro60gkYnJzc01TU1MSq5pamzZtMkVFRWNuO3nypJk9e7Z5+OGHo+t++9vfGkmms7PTGPPJF1dqaqoJBALRNj/+8Y9NRkaGGR4entLap8rnv4BHRkZMTk6O2bJlS3TdyZMnjcfjMb/4xS+MMcb09vYaSaarqyva5oknnjApKSnm7bffNsYY86Mf/cgsWrQoZly+//3vmxUrVkzxESXOeOHka1/72rjvccvYHDt2zEgyzzzzjDEmcX8/3/ve98yVV14Zs6+NGzeaysrKqT6khPn82BjzSTj5zne+M+573DI2ixYtMj/96U/5vIzD9Zd1Tp8+re7ubvn9/ui61NRU+f1+dXZ2JrGyqff6668rNzdXy5cv16233qq33npLktTd3a2PP/44Zky+9KUvKT8/PzomnZ2dWrVqlbxeb7RNZWWlwuGwXnnllek9kCnS39+vQCAQMw6ZmZny+Xwx47Bw4UKVlpZG2/j9fqWmpuqFF16Itrn22muVnp4ebVNZWakjR47o97///TQdzdRob2/X4sWLtWLFCn3rW9/S+++/H93mlrEJhUKSpKysLEmJ+/vp7OyM6eNsGyf9d+nzY3PWz3/+c2VnZ2vlypVqbGzUBx98EN0208cmEolo165dGhoaUkVFBZ+XcTjuh/8S7fjx44pEIjH/6JLk9Xr16quvJqmqqefz+dTS0qIVK1bo3Xff1X333ac1a9bo8OHDCgQCSk9P18KFC2Pe4/V6FQgEJEmBQGDMMTu7bSY4exxjHednx2Hx4sUx22fNmqWsrKyYNpdeeumoPs5uW7Ro0ZTUP9Wuv/563XLLLbr00kv1xhtv6Ac/+IGqqqrU2dmptLQ0V4zNyMiI/uZv/kZ/9Ed/pJUrV0pSwv5+xmsTDof14Ycfau7cuVNxSAkz1thI0l/+5V9q2bJlys3N1aFDh/T9739fR44c0SOPPCJp5o7Nyy+/rIqKCn300UeaP3++9uzZo8LCQvX09PB5GYPrw4lbVVVVRf/36tWr5fP5tGzZMj300EOO+xAjOf7iL/4i+r9XrVql1atX64tf/KLa29u1bt26JFY2ferq6nT48GE999xzyS7FdsYbmzvvvDP6v1etWqUlS5Zo3bp1euONN/TFL35xusucNitWrFBPT49CoZD+67/+S9XV1XrmmWeSXZZtuf6yTnZ2ttLS0kbNjA4Gg8rJyUlSVdNv4cKFuuKKK9TX16ecnBydPn1aJ0+ejGnz2THJyckZc8zObpsJzh7HuT4bOTk5OnbsWMz2M2fO6MSJE64aK0lavny5srOz1dfXJ2nmj81dd92lX/7yl9q3b58uueSS6PpE/f2M1yYjI8P2/wdivLEZi8/nk6SYz81MHJv09HRddtllKikpUVNTk4qKivQv//IvfF7G4fpwkp6erpKSErW1tUXXjYyMqK2tTRUVFUmsbHoNDg7qjTfe0JIlS1RSUqLZs2fHjMmRI0f01ltvRcekoqJCL7/8csyXz1NPPaWMjAwVFhZOe/1T4dJLL1VOTk7MOITDYb3wwgsx43Dy5El1d3dH2zz99NMaGRmJ/ke3oqJCHR0d+vjjj6NtnnrqKa1YscL2ly3i8bvf/U7vv/++lixZImnmjo0xRnfddZf27Nmjp59+etRlqUT9/VRUVMT0cbaNnf+7dL6xGUtPT48kxXxuZuLYfN7IyIiGh4dd/Xk5p2TPyLWDXbt2GY/HY1paWkxvb6+58847zcKFC2NmRs803/3ud017e7vp7+83v/nNb4zf7zfZ2dnm2LFjxphPbm3Lz883Tz/9tDlw4ICpqKgwFRUV0fefvbXtq1/9qunp6TFPPvmkufjiix13K/GpU6fMiy++aF588UUjyWzdutW8+OKL5s033zTGfHIr8cKFC82jjz5qDh06ZL72ta+NeSvxH/7hH5oXXnjBPPfcc+byyy+PuV325MmTxuv1mm984xvm8OHDZteuXWbevHm2vl3WmHOPzalTp8zdd99tOjs7TX9/v/n1r39trrrqKnP55Zebjz76KNrHTBybb33rWyYzM9O0t7fH3A77wQcfRNsk4u/n7K2h99xzj/ntb39rLMuy/a2h5xubvr4+8/d///fmwIEDpr+/3zz66KNm+fLl5tprr432MRPHpqGhwTzzzDOmv7/fHDp0yDQ0NJiUlBTzq1/9yhjj3s/LuRBO/t+DDz5o8vPzTXp6uikvLzfPP/98skuaUhs3bjRLliwx6enpZunSpWbjxo2mr68vuv3DDz80f/3Xf20WLVpk5s2bZzZs2GDefffdmD6OHj1qqqqqzNy5c012drb57ne/az7++OPpPpRJ2bdvn5E0aqmurjbGfHI78d/+7d8ar9drPB6PWbdunTly5EhMH++//775+te/bubPn28yMjJMTU2NOXXqVEybl156yVxzzTXG4/GYpUuXms2bN0/XIU7Yucbmgw8+MF/96lfNxRdfbGbPnm2WLVtmamtrRwX6mTg2Y42JJPOzn/0s2iZRfz/79u0zxcXFJj093SxfvjxmH3Z0vrF56623zLXXXmuysrKMx+Mxl112mbnnnntinnNizMwbmzvuuMMsW7bMpKenm4svvtisW7cuGkyMce/n5VxSjDFm+s7TAAAAnJvr55wAAAB7IZwAAABbIZwAAABbIZwAAABbIZwAAABbIZwAAABbIZwAAABbIZwAAABbIZwAAABbIZwAAABbIZwAAABbIZwAAABb+T9PFdxaNjNyBAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "plt.hist(sizes, bins=100)\n", "plt.yscale(\"log\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 185, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "33\n", "-zmMiRnUdF0 Cala Bejor - Cafe Club Ibiza Chillout\n", "2LHPln337Lk Cala Bejor - Cafe Club Ibiza Chillout\n", "2mlFetExBOA Cala Bejor - Cafe Club Ibiza Chillout\n", "3qPwdhdOJTU Cala Bejor - Cafe Club Ibiza Chillout\n", "3vnMkJSLLkk Cala Bejor - Cafe Club Ibiza Chillout\n", "5zxGEmFnJHs Cala Bejor - Cafe Club Ibiza Chillout\n", "7nMxA-QCJA4 Cala Bejor - Cafe Club Ibiza Chillout\n", "8YEyWSFE5lA Cala Bejor - Cafe Club Ibiza Chillout\n", "H_hQmOwed5w Cala Bejor - Cafe Club Ibiza Chillout\n", "HwGTMYfPYvo Cala Bejor - Cafe Club Ibiza Chillout\n" ] } ], "source": [ "\n", "result = sorted_results[10006]\n", "print(result['size'])\n", "for item in result[\"items\"][:10]:\n", " print(item[\"id\"], item[\"title\"])" ] }, { "cell_type": "code", "execution_count": 190, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idtitlecluster_size
7000086754-0871-42c4-88ff-30fb7e5a19d8- Chopin, Frédéric3197
7001c4755-7fa1-4be3-a117-4c0dcc527866- Bach, Johann Sebastian2726
64006873c3-0234-49d2-aa7e-f5cacaa97fafGoldberg Variations, BWV 988 - Bach, Johann Se...2248
184004653d0-d57b-40f1-b3e1-1185550b16f4Messiah, HWV 56 (17548 version) - Handel, Geor...1918
353831-2KibGueCx8A State of Trance (ASOT 1118) (Coming Up, Pt. ...1760
\n", "
" ], "text/plain": [ " id \\\n", "70 00086754-0871-42c4-88ff-30fb7e5a19d8 \n", "7 001c4755-7fa1-4be3-a117-4c0dcc527866 \n", "64 006873c3-0234-49d2-aa7e-f5cacaa97faf \n", "184 004653d0-d57b-40f1-b3e1-1185550b16f4 \n", "353831 -2KibGueCx8 \n", "\n", " title cluster_size \n", "70 - Chopin, Frédéric 3197 \n", "7 - Bach, Johann Sebastian 2726 \n", "64 Goldberg Variations, BWV 988 - Bach, Johann Se... 2248 \n", "184 Messiah, HWV 56 (17548 version) - Handel, Geor... 1918 \n", "353831 A State of Trance (ASOT 1118) (Coming Up, Pt. ... 1760 " ] }, "execution_count": 190, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "# Create a dataframe from the first titles and cluster sizes\n", "first_titles_data = [(result[\"items\"][0][\"id\"], result[\"items\"][0][\"title\"], result[\"size\"]) for result in results]\n", "first_titles_df = pd.DataFrame(first_titles_data, columns=[\"id\", \"title\", \"cluster_size\"])\n", "\n", "# sort by cluster_size\n", "first_titles_df = first_titles_df.sort_values(by=\"cluster_size\", ascending=False)\n", "\n", "# Save the dataframe to a CSV file\n", "first_titles_df.to_csv(\"/home/christian/code/christian/metadata/dedup/first_titles_clusters.csv\", index=False)\n", "\n", "# Display the first few rows\n", "first_titles_df.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idtitlecluster_size
353831-2KibGueCx8A State of Trance (ASOT 1118) (Coming Up, Pt. ...1760
327369-Aqpx4W58uQA State of Trance (ASOT 1152) (Shout Outs, Pt....744
864299-5P1f1MfRbAA State Of Trance (ASOT 1097) (Discord Giveawa...655
883637-5gt5T37rL4Insomnia Help Crickets - 4D Nature Recordings,...550
885751--CERokrVDgA State Of Trance (ASOT 967) (This Week's Serv...535
886445-D_jK1kul5EHappy Birthday (Normal Leatherette) - Happy Bi...522
867340-0O7oWaQ6MURepeatable Cicadas Sound - 4D Nature Recording...490
875691-0LiK_UMBxIThe Sound of Rain, Pt. 61 - Regengeräusche, Ra...456
922362-8Fmi6QR2twGroup Therapy (Messages Pt. 4) [ABGT392] - Abo...384
897787-Ol6-VNZ40sMeeresrauschen, Pt. 28 - Meeresrauschen zum Sc...379
891603--1ohvKFkfkRelaxation profonde avec bruit blanc, Pt. 38 -...369
912220--UzV8Lh4nAMeeresrauschen, Pt. 75 - Meeresrauschen zum Ei...298
856289-3n8qC_qjFkMeeresrauschen, Pt. 62 - Meeresrauschen Ullric...295
876268-9AjFDOFfnUTinnitus Help Cricket Sound - Cricket Sounds, ...292
860570-cSed5xn4W4Meeresrauschen, Pt. 10 - Meeresrauschen für de...292
864504-5fgTUqk5D4Meeresrauschen, Pt. 97 - Meeresrauschen für to...290
858531-9SvyWC7bCQSea Noises, Pt. 45 - Ocean Sounds by Dominik A...290
860658-2o2ypF1uWAMeeresrauschen, Pt. 83 - Meeresrauschen Aufnah...278
1002857-IkMcBGgxXIMeeresrauschen, Pt. 15 - Meeresrauschen zum sc...278
856967-H2FsinPocMTelevision and Film Sound Track Music 4297 - P...268
\n", "
" ], "text/plain": [ " id title \\\n", "353831 -2KibGueCx8 A State of Trance (ASOT 1118) (Coming Up, Pt. ... \n", "327369 -Aqpx4W58uQ A State of Trance (ASOT 1152) (Shout Outs, Pt.... \n", "864299 -5P1f1MfRbA A State Of Trance (ASOT 1097) (Discord Giveawa... \n", "883637 -5gt5T37rL4 Insomnia Help Crickets - 4D Nature Recordings,... \n", "885751 --CERokrVDg A State Of Trance (ASOT 967) (This Week's Serv... \n", "886445 -D_jK1kul5E Happy Birthday (Normal Leatherette) - Happy Bi... \n", "867340 -0O7oWaQ6MU Repeatable Cicadas Sound - 4D Nature Recording... \n", "875691 -0LiK_UMBxI The Sound of Rain, Pt. 61 - Regengeräusche, Ra... \n", "922362 -8Fmi6QR2tw Group Therapy (Messages Pt. 4) [ABGT392] - Abo... \n", "897787 -Ol6-VNZ40s Meeresrauschen, Pt. 28 - Meeresrauschen zum Sc... \n", "891603 --1ohvKFkfk Relaxation profonde avec bruit blanc, Pt. 38 -... \n", "912220 --UzV8Lh4nA Meeresrauschen, Pt. 75 - Meeresrauschen zum Ei... \n", "856289 -3n8qC_qjFk Meeresrauschen, Pt. 62 - Meeresrauschen Ullric... \n", "876268 -9AjFDOFfnU Tinnitus Help Cricket Sound - Cricket Sounds, ... \n", "860570 -cSed5xn4W4 Meeresrauschen, Pt. 10 - Meeresrauschen für de... \n", "864504 -5fgTUqk5D4 Meeresrauschen, Pt. 97 - Meeresrauschen für to... \n", "858531 -9SvyWC7bCQ Sea Noises, Pt. 45 - Ocean Sounds by Dominik A... \n", "860658 -2o2ypF1uWA Meeresrauschen, Pt. 83 - Meeresrauschen Aufnah... \n", "1002857 -IkMcBGgxXI Meeresrauschen, Pt. 15 - Meeresrauschen zum sc... \n", "856967 -H2FsinPocM Television and Film Sound Track Music 4297 - P... \n", "\n", " cluster_size \n", "353831 1760 \n", "327369 744 \n", "864299 655 \n", "883637 550 \n", "885751 535 \n", "886445 522 \n", "867340 490 \n", "875691 456 \n", "922362 384 \n", "897787 379 \n", "891603 369 \n", "912220 298 \n", "856289 295 \n", "876268 292 \n", "860570 292 \n", "864504 290 \n", "858531 290 \n", "860658 278 \n", "1002857 278 \n", "856967 268 " ] }, "execution_count": 196, "metadata": {}, "output_type": "execute_result" } ], "source": [ "short_ids_df = first_titles_df[first_titles_df['id'].str.len() < 12]\n" ] }, { "cell_type": "code", "execution_count": 201, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idtitlecluster_size
894660-BguAQ92CgoTropical Birdsongs Sounds - Nature Sounds, Cal...142
941176-WH2630yPjILittle Bo Peep - Nursery Lullabyes, Baby Lulla...141
878740-3dW8IAmQlgHealing Flycatchers Sound - Calm Singing Brids...141
932063-EcY2gqBMKoCalming Bird Music - Nature Sounds, Calm Singi...141
985661-MgJ23B2ke0Rock A Bye Baby - Nursery Lullabyes, Baby Lull...140
908386-m0APeXy6cITu es einfach und glaub daran (Teil 40) - Thom...140
876815-8xQiy4ERmoHey Diddle Diddle - Nursery Lullabyes, Baby Lu...139
997917-wnNr6WxrnACalm Nature Recordings, Pt. 15 - Soft Soundsca...139
869770-0Gy97iiUcgNocturnal Wellness White Noise, Pt. 18 - White...139
8647120HPLX-ITL70Calming Waves Sounds - Ocean Sounds, Calming W...138
875809-TBMc75h4S4Easy Listening Crows - Calm Singing Brids Zone...138
927803-UwT4r0L1kwRegen Sounds, Pt. 8 - Regen zum Einschlafen un...137
923042-0NuVwIphv8Group Therapy (Messages Pt. 2) [ABGT357] - Anj...137
896014-0Y-ClSaO90Baa Baa Black Sheep - Nursery Lullabyes, Baby ...136
952458-_EVe8N3YagHeads Shoulders Knees And Toes - Nursery Lulla...136
9253190X8nUVMfPEwRuido Blanco para Dormir 4 - 8 - Pure White Noise136
951413-Pdd4MVesgAOld Macdonald Had A Farm - Nursery Lullabyes, ...135
965786-ar9wd0qR0IHumpty Dumpty - Nursery Lullabyes, Baby Lullab...135
974143-QJ-FMDyT5QBaby Calm Soothing Sounds, Pt. 19 - White Nois...135
1073586-IdYaJ6bXTkElectronic By Swoosh 120 - Alan Paul Ett, Benj...135
\n", "
" ], "text/plain": [ " id title \\\n", "894660 -BguAQ92Cgo Tropical Birdsongs Sounds - Nature Sounds, Cal... \n", "941176 -WH2630yPjI Little Bo Peep - Nursery Lullabyes, Baby Lulla... \n", "878740 -3dW8IAmQlg Healing Flycatchers Sound - Calm Singing Brids... \n", "932063 -EcY2gqBMKo Calming Bird Music - Nature Sounds, Calm Singi... \n", "985661 -MgJ23B2ke0 Rock A Bye Baby - Nursery Lullabyes, Baby Lull... \n", "908386 -m0APeXy6cI Tu es einfach und glaub daran (Teil 40) - Thom... \n", "876815 -8xQiy4ERmo Hey Diddle Diddle - Nursery Lullabyes, Baby Lu... \n", "997917 -wnNr6WxrnA Calm Nature Recordings, Pt. 15 - Soft Soundsca... \n", "869770 -0Gy97iiUcg Nocturnal Wellness White Noise, Pt. 18 - White... \n", "864712 0HPLX-ITL70 Calming Waves Sounds - Ocean Sounds, Calming W... \n", "875809 -TBMc75h4S4 Easy Listening Crows - Calm Singing Brids Zone... \n", "927803 -UwT4r0L1kw Regen Sounds, Pt. 8 - Regen zum Einschlafen un... \n", "923042 -0NuVwIphv8 Group Therapy (Messages Pt. 2) [ABGT357] - Anj... \n", "896014 -0Y-ClSaO90 Baa Baa Black Sheep - Nursery Lullabyes, Baby ... \n", "952458 -_EVe8N3Yag Heads Shoulders Knees And Toes - Nursery Lulla... \n", "925319 0X8nUVMfPEw Ruido Blanco para Dormir 4 - 8 - Pure White Noise \n", "951413 -Pdd4MVesgA Old Macdonald Had A Farm - Nursery Lullabyes, ... \n", "965786 -ar9wd0qR0I Humpty Dumpty - Nursery Lullabyes, Baby Lullab... \n", "974143 -QJ-FMDyT5Q Baby Calm Soothing Sounds, Pt. 19 - White Nois... \n", "1073586 -IdYaJ6bXTk Electronic By Swoosh 120 - Alan Paul Ett, Benj... \n", "\n", " cluster_size \n", "894660 142 \n", "941176 141 \n", "878740 141 \n", "932063 141 \n", "985661 140 \n", "908386 140 \n", "876815 139 \n", "997917 139 \n", "869770 139 \n", "864712 138 \n", "875809 138 \n", "927803 137 \n", "923042 137 \n", "896014 136 \n", "952458 136 \n", "925319 136 \n", "951413 135 \n", "965786 135 \n", "974143 135 \n", "1073586 135 " ] }, "execution_count": 201, "metadata": {}, "output_type": "execute_result" } ], "source": [ "short_ids_df.iloc[120:140]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import re\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.cluster import DBSCAN\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "\n", "# Sample data\n", "data = [\n", " {\"id\": 1, \"title\": \"Coldplay - Clocks (Official Video)\"},\n", " {\"id\": 2, \"title\": \"Clocks - Coldplay\"},\n", " {\"id\": 3, \"title\": \"Coldplay - Clocks, Live\"},\n", " {\"id\": 4, \"title\": \"Coldplay: Clocks [Remix]\"},\n", " {\"id\": 5, \"title\": \"clocks by coldplay (HD)\"},\n", " {\"id\": 6, \"title\": \"Bohemian Rhapsody - Queen\"},\n", " {\"id\": 7, \"title\": \"Queen - Bohemian Rhapsody (Official Video)\"},\n", "]\n", "\n", "data = titles\n", "\n", "df = pd.DataFrame(data)\n", "\n", "# Clean and normalize titles\n", "def clean_title(title):\n", " title = title.lower()\n", " title = re.sub(r'\\([^)]*\\)', '', title)\n", " title = re.sub(r'\\[[^]]*\\]', '', title)\n", " #title = re.sub(r'[^a-z0-9\\s]', '', title)\n", " title = re.sub(r'\\s+', ' ', title).strip()\n", " return title\n", "\n", "df['cleaned'] = df['title'].apply(clean_title)\n", "\n", "# Vectorize using TF-IDF\n", "vectorizer = TfidfVectorizer()\n", "tfidf_matrix = vectorizer.fit_transform(df['cleaned'])\n", "\n", "# Cosine similarity and distance\n", "similarity_matrix = cosine_similarity(tfidf_matrix)\n", "\n", "# Compute distance matrix and clip to [0, 1] to prevent negative values\n", "distance_matrix = 1 - similarity_matrix\n", "distance_matrix = np.clip(distance_matrix, 0, 1)\n", "\n", "# DBSCAN clustering\n", "clustering = DBSCAN(eps=0.05, min_samples=2, metric='precomputed')\n", "df['cluster'] = clustering.fit_predict(distance_matrix)\n", "\n", "# Display results\n", "# Count the number of items in each cluster\n", "cluster_counts = df[df['cluster'] != -1]['cluster'].value_counts().sort_values(ascending=False)\n", "\n", "# Display results sorted by cluster size\n", "for cluster_id in cluster_counts.index:\n", " group = df[df['cluster'] == cluster_id]\n", " print(f\"\\nGroup {cluster_id} ({len(group)} matches):\")\n", " for _, row in group.iterrows():\n", " print(f\" - ID {row['id']}: {row['title']}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Parallel" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "import re\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.cluster import DBSCAN\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "from joblib import Parallel, delayed\n", "from tqdm import tqdm\n", "import hdbscan\n", "\n", "\n", "\n", "data = titles#[:1000]\n", "\n", "df = pd.DataFrame(data)\n", "\n", "# Clean and normalize titles\n", "def clean_title(title):\n", " title = title.lower()\n", " title = re.sub(r'\\([^)]*\\)', '', title)\n", " title = re.sub(r'\\[[^]]*\\]', '', title)\n", " #title = re.sub(r'[^a-z0-9\\s]', '', title)\n", " title = re.sub(r'\\s+', ' ', title).strip()\n", " return title\n", "\n", "df['cleaned'] = df['title'].apply(clean_title)\n", "\n", "# Vectorize using TF-IDF\n", "print(\"making tfidf matrix\")\n", "vectorizer = TfidfVectorizer()\n", "tfidf_matrix = vectorizer.fit_transform(df['cleaned'])\n", "\n", "# Parallel processing across rows\n", "n_jobs = -1 # use all CPUs\n", "print(\"making similiarity matrix\")\n", "\n", "def compute_batch(start, end, matrix):\n", " batch = matrix[start:end]\n", " return (start, end, cosine_similarity(batch, matrix))\n", "\n", "def batched_cosine_similarity_parallel(matrix, batch_size=100, n_jobs=-1):\n", " n = matrix.shape[0]\n", " batch_indices = [(i, min(i + batch_size, n)) for i in range(0, n, batch_size)]\n", "\n", " # Parallel compute batches\n", " results = Parallel(n_jobs=64)(\n", " delayed(compute_batch)(start, end, matrix)\n", " for start, end in tqdm(batch_indices, desc=\"Computing similarity in parallel\")\n", " )\n", "\n", " # Build final similarity matrix\n", " similarity_matrix = np.zeros((n, n))\n", " for start, end, sim_block in results:\n", " similarity_matrix[start:end] = sim_block\n", "\n", " return similarity_matrix\n", "\n", "def batched_cosine_similarity(matrix, batch_size=100):\n", " n = matrix.shape[0]\n", " similarity_matrix = np.zeros((n, n))\n", " \n", " for i in tqdm(range(0, n, batch_size), desc=\"Computing similarity in batches\"):\n", " i_end = min(i + batch_size, n)\n", " batch = matrix[i:i_end]\n", " \n", " # Compute similarity of this batch vs full matrix\n", " sim_block = cosine_similarity(batch, matrix)\n", " similarity_matrix[i:i_end] = sim_block\n", " \n", " return similarity_matrix\n", "\n", "#similarity_matrix = batched_cosine_similarity(tfidf_matrix)\n", "similarity_matrix = batched_cosine_similarity_parallel(tfidf_matrix)\n", "\n", "print(\"computing distance\")\n", "# Compute distance matrix and clip to [0, 1] to prevent negative values\n", "\n", "distance_matrix = 1 - similarity_matrix\n", "distance_matrix = np.clip(distance_matrix, 0, 1)\n", "\n", "# DBSCAN clustering\n", "print(\"clustering\")\n", "#clustering = DBSCAN(eps=0.05, min_samples=2, metric='precomputed', n_jobs=-1)\n", "clusterer = hdbscan.HDBSCAN(min_cluster_size=10)\n", "df['cluster'] = clusterer.fit_predict(distance_matrix)\n", "#df['cluster'] = clustering.fit_predict(distance_matrix)\n", "\n", "# Display results\n", "# Count the number of items in each cluster\n", "cluster_counts = df[df['cluster'] != -1]['cluster'].value_counts().sort_values(ascending=False)\n", "\n", "# Display results sorted by cluster size\n", "for cluster_id in cluster_counts.index:\n", " group = df[df['cluster'] == cluster_id]\n", " print(f\"\\nGroup {cluster_id} ({len(group)} matches):\")\n", " for _, row in group.iterrows():\n", " print(f\" - ID {row['id']}: {row['title']}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import faiss\n", "import re\n", "import pandas as pd\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.preprocessing import normalize\n", "\n", "# 1. Sample titles\n", "df = pd.DataFrame(titles[:50000])\n", "\n", "# Clean and normalize titles\n", "def clean_title(title):\n", " title = title.lower()\n", " title = re.sub(r'\\([^)]*\\)', '', title)\n", " title = re.sub(r'\\[[^]]*\\]', '', title)\n", " #title = re.sub(r'[^a-z0-9\\s]', '', title)\n", " title = re.sub(r'\\s+', ' ', title).strip()\n", " return title\n", "\n", "df['cleaned'] = df['title'].apply(clean_title)\n", "\n", "# Vectorize using TF-IDF\n", "print(\"making tfidf matrix\")\n", "vectorizer = TfidfVectorizer()\n", "X = vectorizer.fit_transform(df['cleaned'])\n", "\n", "# 3. Normalize vectors (cosine similarity simulation)\n", "X_normalized = normalize(X, norm='l2', axis=1)\n", "\n", "# 4. FAISS index using inner product for cosine\n", "dim = X_normalized.shape[1]\n", "index = faiss.IndexFlatIP(dim)\n", "index.add(X_normalized.toarray())\n" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "\n", "# 5. Search top-k neighbors (excluding self-match)\n", "k = 50\n", "D, I = index.search(X_normalized.toarray(), k)\n", "\n", "# 6. Filter by similarity threshold and collect matches\n", "threshold = 0.95\n", "pairs = []\n", "for i, (neighbors, sims) in enumerate(zip(I, D)):\n", " for j_idx, sim in zip(neighbors[1:], sims[1:]): # skip self-match\n", " if sim >= threshold and df.loc[i, 'id'] != df.loc[j_idx, 'id']: # exclude exact ID matches\n", " pairs.append({\n", " 'id_1': df.loc[i, 'id'],\n", " 'id_2': df.loc[j_idx, 'id'],\n", " 'title_1': df.loc[i, 'title'],\n", " 'title_2': df.loc[j_idx, 'title'],\n", " 'similarity': sim\n", " })\n", "\n", "# 7. Convert to DataFrame\n", "similar_df = pd.DataFrame(pairs)\n", "similar_df.sort_values('similarity', ascending=False, inplace=True)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "# 5. Search top-k neighbors (excluding self-match)\n", "k = 50\n", "D, I = index.search(X_normalized.toarray(), k)\n", "\n", "# FAISS output: I (indices), D (similarities)\n", "n_queries, k = I.shape\n", "\n", "# Create arrays of query indices (i), neighbor indices (j), and similarities\n", "i_idx = np.repeat(np.arange(n_queries), k - 1)\n", "j_idx = I[:, 1:].reshape(-1)\n", "sims = D[:, 1:].reshape(-1)\n", "\n", "# Filter by threshold\n", "mask = sims >= threshold\n", "i_idx = i_idx[mask]\n", "j_idx = j_idx[mask]\n", "sims = sims[mask]\n", "\n", "# Map back to DataFrame\n", "similar_df = pd.DataFrame({\n", " 'id_1': df.iloc[i_idx]['id'].values,\n", " 'title_1': df.iloc[i_idx]['title'].values,\n", " 'id_2': df.iloc[j_idx]['id'].values,\n", " 'title_2': df.iloc[j_idx]['title'].values,\n", " 'similarity': sims\n", "})\n", "\n", "# Optional: sort or save\n", "similar_df.sort_values('similarity', ascending=False, inplace=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "similar_df" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [], "source": [ "import re\n", "import unicodedata\n", "import hashlib\n", "from datasketch import MinHash, MinHashLSH\n", "from collections import defaultdict\n", "\n", "# -------------------------------\n", "# 1. Normalization\n", "# -------------------------------\n", "\n", "def normalize(text):\n", " # Unicode normalize + casefold\n", " text = unicodedata.normalize(\"NFKD\", text).casefold()\n", " text = \"\".join(c for c in text if not unicodedata.combining(c))\n", "\n", " # Remove bracketed content like \"(Official Video)\", \"[Remix]\"\n", " text = re.sub(r\"\\([^)]*\\)|\\[[^]]*\\]\", \" \", text)\n", "\n", " # REMOVE common noise words early\n", " text = re.sub(\n", " r\"\\b(remaster(ed)?|live|mono|stereo|version|edit|single|official|video|hd|feat\\.?|intro|outro|instrumental|radio|mix|remix)\\b\",\n", " \" \",\n", " text,\n", " )\n", "\n", "\n", " # Collapse non-alphanumerics\n", " #text = re.sub(r\"[^0-9\\p{L}]+\", \" \", text, flags=re.UNICODE)\n", " return \" \".join(text.split())\n", "\n", "# -------------------------------\n", "# 2. Blocking key\n", "# -------------------------------\n", "\n", "def block_key(text, size=6):\n", " # Strip vowels, take first few consonants\n", " consonants = re.sub(r\"[aeiou\\s]\", \"\", text)\n", " return hashlib.md5(consonants[:size].encode()).hexdigest()[:8]\n", "\n", "# -------------------------------\n", "# 3. Char n-grams for Jaccard\n", "# -------------------------------\n", "\n", "def char_ngrams(s, n=3):\n", " s = f\" {s} \"\n", " return {s[i:i+n] for i in range(len(s) - n + 1)}\n", "\n", "# -------------------------------\n", "# 4. MinHash signature\n", "# -------------------------------\n", "\n", "def create_minhash(s, num_perm=128):\n", " m = MinHash(num_perm=num_perm)\n", " for g in char_ngrams(s):\n", " m.update(g.encode(\"utf8\"))\n", " return m\n", "\n", "# -------------------------------\n", "# 5. Union-Find\n", "# -------------------------------\n", "\n", "class UnionFind:\n", " def __init__(self):\n", " self.parent = dict()\n", "\n", " def find(self, x):\n", " if x not in self.parent: self.parent[x] = x\n", " if self.parent[x] != x:\n", " self.parent[x] = self.find(self.parent[x])\n", " return self.parent[x]\n", "\n", " def union(self, x, y):\n", " self.parent[self.find(x)] = self.find(y)\n", "\n", "# -------------------------------\n", "# 6. Main clustering function\n", "# -------------------------------\n", "\n", "def cluster_records(data, jaccard_threshold=0.99, lsh_threshold=0.98):\n", " # Step 1: normalize and prepare\n", " norm_map = {d[\"id\"]: normalize(d[\"title\"]) for d in data}\n", " blocks = defaultdict(list)\n", " for id_, text in norm_map.items():\n", " blocks[block_key(text)].append((id_, text))\n", "\n", " uf = UnionFind()\n", "\n", " # Step 2: process each block\n", " for block in blocks.values():\n", " if len(block) <= 1:\n", " continue\n", "\n", " lsh = MinHashLSH(threshold=lsh_threshold, num_perm=128)\n", " sigs = dict()\n", "\n", " # Build LSH index\n", " for id_, text in block:\n", " mh = create_minhash(text)\n", " lsh.insert(id_, mh)\n", " sigs[id_] = (text, mh)\n", "\n", " # Query and verify\n", " for id1, (text1, mh1) in sigs.items():\n", " for id2 in lsh.query(mh1):\n", " if id1 >= id2:\n", " continue # avoid dupes\n", " text2 = sigs[id2][0]\n", " jaccard = len(char_ngrams(text1) & char_ngrams(text2)) / len(char_ngrams(text1) | char_ngrams(text2))\n", " if jaccard >= jaccard_threshold:\n", " uf.union(id1, id2)\n", "\n", " # Step 3: extract clusters\n", " clusters = defaultdict(list)\n", " for id_ in norm_map:\n", " clusters[uf.find(id_)].append(id_)\n", "\n", " return [sorted(cluster) for cluster in clusters.values() if len(cluster) > 1]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "clusters = cluster_records(titles[:100000])\n", "print(clusters)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(f\"Found {len(clusters)} clusters\")\n", "for cluster in clusters:\n", " print(len(cluster))\n", " for id_ in cluster:\n", " print(id_, id2title[id_])\n", " print(\"-\"*100)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Parallel attempt 2" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "from concurrent.futures import ProcessPoolExecutor, as_completed\n", "from tqdm import tqdm\n", "from datasketch import MinHash, MinHashLSH\n", "from collections import defaultdict\n", "\n", "# Reuse these from before\n", "def char_ngrams(s, n=3):\n", " s = f\" {s} \"\n", " return {s[i:i+n] for i in range(len(s) - n + 1)}\n", "\n", "def create_minhash(ngrams, num_perm=128):\n", " m = MinHash(num_perm=num_perm)\n", " for g in ngrams:\n", " m.update(g.encode(\"utf8\"))\n", " return m\n", "\n", "def process_block(block, lsh_thresh=0.85, jaccard_thresh=0.92, num_perm=128):\n", " pairs = set()\n", " if len(block) <= 1:\n", " return pairs\n", "\n", " lsh = MinHashLSH(threshold=lsh_thresh, num_perm=num_perm)\n", " ngram_sets = {}\n", " sigs = {}\n", "\n", " # Build LSH\n", " for id_, text in block:\n", " ngrams = char_ngrams(text)\n", " ngram_sets[id_] = ngrams\n", " sig = create_minhash(ngrams, num_perm)\n", " sigs[id_] = sig\n", " lsh.insert(id_, sig)\n", "\n", " # Query and filter\n", " for id1 in sigs:\n", " for id2 in lsh.query(sigs[id1]):\n", " if id1 >= id2:\n", " continue\n", " a, b = ngram_sets[id1], ngram_sets[id2]\n", " sim = len(a & b) / len(a | b)\n", " if sim >= jaccard_thresh:\n", " pairs.add((id1, id2))\n", "\n", " return pairs\n", "\n", "def cluster_records_parallel(data, jaccard_threshold=0.92, lsh_threshold=0.95):\n", "\n", " norm_map = {d[\"id\"]: normalize(d[\"title\"]) for d in data}\n", " blocks = defaultdict(list)\n", " for id_, text in norm_map.items():\n", " blocks[block_key(text)].append((id_, text))\n", "\n", " uf = UnionFind()\n", "\n", " # Parallel block processing\n", " all_pairs = []\n", " with ProcessPoolExecutor() as executor:\n", " futures = {\n", " executor.submit(process_block, block, lsh_threshold, jaccard_threshold): block\n", " for block in blocks.values()\n", " }\n", " for future in tqdm(as_completed(futures), total=len(futures), desc=\"Clustering blocks\"):\n", " result = future.result()\n", " all_pairs.extend(result)\n", "\n", " # Union-Find\n", " for id1, id2 in all_pairs:\n", " uf.union(id1, id2)\n", "\n", " # Collect clusters\n", " clusters = defaultdict(list)\n", " for id_ in norm_map:\n", " clusters[uf.find(id_)].append(id_)\n", "\n", " return [sorted(cluster) for cluster in clusters.values() if len(cluster) > 1]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "clusters = cluster_records(titles[:100000])\n", "print(clusters)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(f\"Found {len(clusters)} clusters\")\n", "for cluster in clusters:\n", " print(len(cluster))\n", " for id_ in cluster:\n", " print(id_, id2title[id_])\n", " print(\"-\"*100)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }