{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import json\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "from rapidfuzz import fuzz\n",
    "from suno_utils.utils.text import read_jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2090009/2090009 [00:58<00:00, 35639.93it/s]\n"
     ]
    }
   ],
   "source": [
    "# load genius metas\n",
    "genius_metas = read_jsonl(\"/home/christian/code/christian/metadata/genius_hq_metas.jsonl\", progress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read csv file into a list of dictionaries\n",
    "def read_csv(file_path):\n",
    "    with open(file_path, mode='r') as file:\n",
    "        reader = csv.DictReader(file)\n",
    "        return [row for row in reader]\n",
    "\n",
    "spot_metas = read_csv(\"/home/christian/code/christian/metadata/spot_genres.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "114000\n"
     ]
    }
   ],
   "source": [
    "print(len(spot_metas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Unnamed: 0': '10', 'track_id': '4mzP5mHkRvGxdhdGdAH7EJ', 'artists': 'Zack Tabudlo', 'album_name': 'Episode', 'track_name': 'Give Me Your Forever', 'popularity': '74', 'duration_ms': '244800', 'explicit': 'False', 'danceability': '0.627', 'energy': '0.363', 'key': '8', 'loudness': '-8.127', 'mode': '1', 'speechiness': '0.0291', 'acousticness': '0.279', 'instrumentalness': '0.0', 'liveness': '0.0928', 'valence': '0.301', 'tempo': '99.905', 'time_signature': '4', 'track_genre': 'acoustic'}\n"
     ]
    }
   ],
   "source": [
    "print(spot_metas[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29860\n"
     ]
    }
   ],
   "source": [
    "# create set of artists from spot_metas\n",
    "artist_set = set()\n",
    "for meta in spot_metas:\n",
    "    artists = meta['artists'].split(\";\")\n",
    "    artist_set.update(artists)\n",
    "\n",
    "print(len(artist_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Rachel Mcalpine\n",
      "Deekapz\n",
      "Orgy\n",
      "Zany\n",
      "Soothing White Noise for Sleeping Babies\n",
      "Pratik Studio\n",
      "3LAU\n",
      "The Peep Tempel\n",
      "Phats & Small\n"
     ]
    }
   ],
   "source": [
    "artist_list = list(artist_set)\n",
    "for n in range(10):\n",
    "    print(artist_list[n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26410\n",
      "rachel-mcalpine\n",
      "deekapz\n",
      "soothing-white-noise-for-sleeping-babies\n",
      "pratik-studio\n",
      "the-peep-tempel\n",
      "phats-&-small\n",
      "pascow\n",
      "blossom-dearie\n",
      "kids-tv-123\n",
      "remo-girone\n"
     ]
    }
   ],
   "source": [
    "# clean artist list\n",
    "artist_list = [artist.strip() for artist in artist_list]\n",
    "artist_list = [artist for artist in artist_list if artist != \"\"]\n",
    "artist_list = [artist for artist in artist_list if artist != \"Various Artists\"]\n",
    "# ensure length of at least 3 characters\n",
    "artist_list = [artist for artist in artist_list if len(artist) > 5]\n",
    "# ensure artist is not a number\n",
    "artist_list = [artist for artist in artist_list if not artist.isnumeric()]\n",
    "# ensure lowercase\n",
    "artist_list = [artist.lower() for artist in artist_list]\n",
    "\n",
    "# add \"-\" between spaces in artist names\n",
    "artist_list = [artist.replace(\" \", \"-\") for artist in artist_list]\n",
    "\n",
    "print(len(artist_list))\n",
    "\n",
    "for n in range(10):\n",
    "    print(artist_list[n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save artist to file\n",
    "with open(\"/home/christian/code/christian/metadata/spot_artist_list_filtered.txt\", \"w\") as file:\n",
    "    for artist in artist_list:\n",
    "        file.write(artist + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now filter genius metas for artists in artist_set\n",
    "genius_metas_filtered = []\n",
    "\n",
    "# use set for faster lookup\n",
    "artist_set = set(artist_list)\n",
    "pbar = tqdm(genius_metas)\n",
    "\n",
    "for meta in pbar:\n",
    "    video_views = meta[\"youtube_views\"]\n",
    "\n",
    "    # first filter on views\n",
    "    if video_views < 100_000:\n",
    "        continue\n",
    "\n",
    "    genius_slug = meta[\"genius_slug\"]\n",
    "    for artist in artist_set:\n",
    "        if fuzz.partial_ratio(artist, genius_slug) > 90:\n",
    "            genius_metas_filtered.append(meta)\n",
    "            print(f\"Found {artist} in {genius_slug}\")\n",
    "            pbar.set_description(f\"Found {len(genius_metas_filtered)}\")\n",
    "            break\n",
    "\n",
    "print(len(genius_metas_filtered))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 11/2091 batches. Estimated remaining time: 92.37 minutes\n",
      "Processed 21/2091 batches. Estimated remaining time: 77.45 minutes\n",
      "Processed 31/2091 batches. Estimated remaining time: 68.88 minutes\n",
      "Processed 41/2091 batches. Estimated remaining time: 63.58 minutes\n",
      "Processed 51/2091 batches. Estimated remaining time: 65.44 minutes\n",
      "Processed 61/2091 batches. Estimated remaining time: 63.46 minutes\n",
      "Processed 71/2091 batches. Estimated remaining time: 64.01 minutes\n",
      "Processed 81/2091 batches. Estimated remaining time: 64.48 minutes\n",
      "Processed 91/2091 batches. Estimated remaining time: 62.91 minutes\n",
      "Processed 101/2091 batches. Estimated remaining time: 63.01 minutes\n"
     ]
    }
   ],
   "source": [
    "from concurrent.futures import ProcessPoolExecutor, as_completed\n",
    "import time\n",
    "\n",
    "# Assuming genius_metas is your list of metadata\n",
    "# Assuming artist_list is your list of artists\n",
    "\n",
    "# Use set for faster lookup\n",
    "artist_set = set(artist_list)\n",
    "\n",
    "def filter_meta_batch(metas, artist_set):\n",
    "    \"\"\"Process a batch of metas.\"\"\"\n",
    "    filtered = []\n",
    "    for meta in metas:\n",
    "        video_views = meta[\"youtube_views\"]\n",
    "\n",
    "        # First filter on views\n",
    "        if video_views < 100_000:\n",
    "            continue\n",
    "\n",
    "        genius_slug = meta[\"genius_slug\"]\n",
    "        for artist in artist_set:\n",
    "            if fuzz.partial_ratio(artist, genius_slug) > 90:\n",
    "                filtered.append(meta)\n",
    "                break\n",
    "    return filtered\n",
    "\n",
    "def print_progress(start_time, processed, total):\n",
    "    elapsed_time = time.time() - start_time\n",
    "    estimated_total_time = (elapsed_time / processed) * total\n",
    "    remaining_time = estimated_total_time - elapsed_time\n",
    "    print(f\"Processed {processed}/{total} batches. Estimated remaining time: {remaining_time / 60:.2f} minutes\", flush=True)\n",
    "\n",
    "# Batching the genius_metas list\n",
    "batch_size = 1000  # Adjust this value as needed\n",
    "genius_metas_batches = [genius_metas[i:i + batch_size] for i in range(0, len(genius_metas), batch_size)]\n",
    "\n",
    "# Create a ProcessPoolExecutor\n",
    "genius_metas_filtered = []\n",
    "start_time = time.time()\n",
    "\n",
    "with ProcessPoolExecutor(max_workers=8) as executor:  # Adjust max_workers based on your CPU/core count\n",
    "    futures = {executor.submit(filter_meta_batch, batch, artist_set): batch for batch in genius_metas_batches}\n",
    "\n",
    "    for i, future in enumerate(as_completed(futures)):\n",
    "        result = future.result()\n",
    "        if result:\n",
    "            genius_metas_filtered.extend(result)\n",
    "\n",
    "        # Print progress after each future is processed\n",
    "        if i % 10 == 0 and i > 0:  # Print every 10 batches\n",
    "            print_progress(start_time, i + 1, len(futures))\n",
    "\n",
    "# Final progress printout\n",
    "print_progress(start_time, len(futures), len(futures))\n",
    "\n",
    "# The filtered list of genius metas\n",
    "print(f\"Total items found: {len(genius_metas_filtered)}\", flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(genius_metas_filtered), len(genius_metas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand_idx = np.random.randint(0, len(genius_metas_filtered))\n",
    "item = genius_metas_filtered[rand_idx]\n",
    "for key, value in item.items():\n",
    "    if key == \"lyrics\":\n",
    "        continue\n",
    "    print(key, value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}