{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import polars as pl\n",
    "\n",
    "from suno_utils.utils.text import read_jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#discogs_subset_filepath = \"/app/suno/tmp/raw_deezer_metas.jsonl\"\n",
    "discogs_subset_filepath = \"/home/christian/code/christian/metadata/combined_v3_w_extreme_metas_v0_aligned.jsonl\"\n",
    "\n",
    "metas = read_jsonl(discogs_subset_filepath)\n",
    "print(len(metas))\n",
    "#df = pl.read_ndjson(discogs_subset_filepath)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metas_tr_filepath = \"/app2/suno/data/auk_v0/metas_v9_tr.jsonl\"\n",
    "metas_tr = read_jsonl(metas_tr_filepath)\n",
    "print(len(metas_tr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to parquet for faster future loading\n",
    "parquet_filepath = \"/app2/suno/data/christian/metadata/metas_v9_tr.parquet\"\n",
    "\n",
    "# Instead of casting everything to string, set strict=False to allow mixed types\n",
    "df_tr = pl.DataFrame(metas_tr, strict=False)\n",
    "df_tr.write_parquet(parquet_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parquet_filepath = \"/app2/suno/data/christian/metadata/combined_v3_w_extreme_metas_v0_aligned.parquet\"\n",
    "\n",
    "# Save to parquet for faster future loading\n",
    "df = pl.DataFrame(metas)\n",
    "df.write_parquet(parquet_filepath)\n",
    "\n",
    "# In the future, load with:\n",
    "# df = pl.read_parquet(parquet_filepath)\n",
    "# count the number of metas\n",
    "print(len(metas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parquet_filepath = \"/app2/suno/data/christian/metadata/raw_discogs_metas.parquet\"\n",
    "parquet_filepath = \"/app2/suno/data/christian/metadata/metas_v9_tr.parquet\"\n",
    "\n",
    "df = pl.read_parquet(parquet_filepath)\n",
    "# sort the df by views\n",
    "#df = df.sort(\"views\", descending=True)\n",
    "# get the first 10\n",
    "#df = df.head(10)\n",
    "#metas_tr_map = {meta[\"id\"]: meta for meta in metas_tr}\n",
    "#\n",
    "# create this metas_tr_map from df\n",
    "#metas_tr_map = {row[\"id\"]: row for row in df.iter_rows(named=True)}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 181\n",
    "meta_id = df.row(idx)[0]\n",
    "\n",
    "df.row(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "# iterate over the rows of the df\n",
    "for idx, row in enumerate(df.iter_rows(named=True)):\n",
    "    meta_id = row[\"id\"]\n",
    "    tr_meta = metas_tr_map[meta_id]\n",
    "    tags = tr_meta.get(\"tags\", [])\n",
    "    text = tr_meta.get(\"text\", \"\")\n",
    "    lang = tr_meta.get(\"lang\", \"\")\n",
    "\n",
    "    # remove any tags that have : in them\n",
    "    tags = [tag for tag in tags if \":\" not in tag]\n",
    "    print(idx, meta_id)\n",
    "    print(f\"tags ({len(tags)}): {tags}\")\n",
    "    print(f\"lang: {lang}\")\n",
    "    # max tag length\n",
    "    if len(tags) > 0:\n",
    "        max_tag_length = max(len(tag) for tag in tags)\n",
    "        print(\"max tag length\", max_tag_length)\n",
    "    else:\n",
    "        max_tag_length = 0\n",
    "    count += 1\n",
    "\n",
    "    has_text = len(text) > 0\n",
    "    print(f\"has_text: {has_text}\")\n",
    "    print()\n",
    "    if count > 300:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_id = \"Ew8T9MRCJuU\"\n",
    "tr_meta = metas_tr_map[meta_id]\n",
    "tags = tr_meta.get(\"tags\", [])\n",
    "text = tr_meta.get(\"text\", \"\")\n",
    "#lyrics = tr_meta.get(\"lyrics\", \"\")\n",
    "lang = tr_meta.get(\"lang\", \"\")\n",
    "weight = tr_meta.get(\"weight\", 0)\n",
    "local_filepath = tr_meta.get(\"local_filepath\", \"\")\n",
    "print(\"lang\", lang)\n",
    "print(tags)\n",
    "print(text)\n",
    "#print(lyrics)\n",
    "print(\"weight\", weight)\n",
    "\n",
    "#from suno_utils.audio import Audio\n",
    "#Audio.from_file(local_filepath).play()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(10):\n",
    "    title = metas[i][\"title\"]\n",
    "    artist = metas[i][\"artists\"][0][\"name\"]\n",
    "    print(f\"\"\"{artist} - \"{title}\" \"\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_terms = [\n",
    "    \"Traduction Française\",         # French\n",
    "    \"Türkçe Çeviri\",                # Turkish\n",
    "    \"Tradução Em Português\",        # Portuguese\n",
    "    \"Traduzione Italiana\",          # Italian\n",
    "    \"Traducción Al Español\",        # Spanish\n",
    "    \"Polskie Tłumaczenie (Polish Translation)\"\n",
    "\n",
    "    \"Deutsche Übersetzung\",         # German\n",
    "    \"日本語翻訳\",                     # Japanese\n",
    "    \"中文翻译\",                      # Chinese (Simplified)\n",
    "    \"繁體中文翻譯\",                  # Chinese (Traditional)\n",
    "    \"한국어 번역\",                   # Korean\n",
    "    \"Русский Перевод\",             # Russian\n",
    "    \"ترجمة عربية\",                  # Arabic\n",
    "    \"हिंदी अनुवाद\",                # Hindi\n",
    "    \"Indonesian Translation\",       # Indonesian (common English form)\n",
    "    \"Terjemahan Bahasa Melayu\",     # Malay\n",
    "    \"ภาษาไทยแปล\",                  # Thai\n",
    "    \"Bản Dịch Tiếng Việt\",          # Vietnamese\n",
    "    \"Tradução Em Galego\",           # Galician\n",
    "    \"Traducció Al Català\",          # Catalan\n",
    "    \"Traducere În Română\",          # Romanian\n",
    "    \"Μετάφραση Στα Ελληνικά\",       # Greek\n",
    "    \"Tłumaczenie Na Polski\",        # Polish\n",
    "    \"Översättning På Svenska\",      # Swedish\n",
    "    \"Fordítás Magyar\",              # Hungarian\n",
    "    \"Překlad Do Češtiny\",           # Czech\n",
    "    \"Preklad Do Slovenčiny\",        # Slovak\n",
    "    \"Prevod Na Srpski\",             # Serbian\n",
    "    \"Překlad Do Slovenštiny\",       # Slovenian\n",
    "    \"Tarjima O‘zbek Tilida\",        # Uzbek\n",
    "    \"Переклад Українською\",         # Ukrainian\n",
    "    \"Traducción Al Euskera\",        # Basque\n",
    "    \"Tradução Em Esperanto\",        # Esperanto\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "suspect_ids = []\n",
    "trigger_term_counter = Counter()\n",
    "triggered_by = defaultdict(list)  # meta_id -> list of trigger terms\n",
    "\n",
    "for meta in tqdm(metas_tr):\n",
    "    tags = meta.get(\"tags\", [])\n",
    "    triggers = [term for term in search_terms if term in tags]\n",
    "    if triggers:\n",
    "        suspect_ids.append(meta[\"id\"])\n",
    "        for term in triggers:\n",
    "            trigger_term_counter[term] += 1\n",
    "        triggered_by[meta[\"id\"]] = triggers\n",
    "\n",
    "print(len(suspect_ids))\n",
    "print(\"Trigger term counts:\")\n",
    "for term, count in trigger_term_counter.most_common():\n",
    "    print(f\"{term}: {count}\")\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random_suspect_id = random.choice(suspect_ids)\n",
    "print(random_suspect_id)\n",
    "meta = metas_tr_map[random_suspect_id]\n",
    "tags = meta.get(\"tags\", [])\n",
    "text = meta.get(\"text\", \"\")\n",
    "lang = meta.get(\"lang\", \"\")\n",
    "print(f\"lang: {lang}\")\n",
    "print(f\"tags: {tags}\")\n",
    "print(f\"text: {text}\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "terms_set = [\n",
    "    \"audiophile\",\n",
    "    \"remix\",\n",
    "    \"cover\",\n",
    "    \"live\",\n",
    "    \"instrumental\",\n",
    "    \"karaoke\",\n",
    "    \"slowed + reverb\"\n",
    "    \"official video\",\n",
    "    \"official audio\",\n",
    "    \"visualizer\",\n",
    "    \"lyrics\",\n",
    "    \"lyric video\",\n",
    "    \"extended\",\n",
    "    \"edit\",\n",
    "    \"radio edit\",\n",
    "    \"club mix\",\n",
    "    \"bootleg\",\n",
    "    \"rework\",\n",
    "    \"re-edit\",\n",
    "    \"reimagined\",\n",
    "    \"demo\",\n",
    "    \"acoustic\",\n",
    "    \"unplugged\",\n",
    "    \"reaction\",\n",
    "    \"performance\",\n",
    "    \"session\",\n",
    "    \"mix\",\n",
    "    \"dj set\",\n",
    "    \"set\",\n",
    "    \"tribute\",\n",
    "    \"reprise\",\n",
    "    \"duet\",\n",
    "    \"mashup\",\n",
    "    \"medley\",\n",
    "    \"fan made\",\n",
    "    \"fan edit\",\n",
    "    \"loop\",\n",
    "    \"8d audio\",\n",
    "    \"360 audio\",\n",
    "    \"slowed\",\n",
    "    \"reverb\",\n",
    "    \"sped up\",\n",
    "    \"nightcore\",\n",
    "    \"bass boosted\",\n",
    "    \"bass-boosted\",\n",
    "    \"hq\",\n",
    "    \"hd\",\n",
    "    \"4k\",\n",
    "    \"vinyl\",\n",
    "    \"vinyl rip\",\n",
    "    \"cassette rip\",\n",
    "    \"tape rip\",\n",
    "    \"full album\",\n",
    "    \"tracklist\",\n",
    "    \"snippet\",\n",
    "    \"teaser\",\n",
    "    \"preview\",\n",
    "    \"leak\",\n",
    "    \"original mix\",\n",
    "    \"alternate version\",\n",
    "    \"studio version\",\n",
    "    \"live version\",\n",
    "    \"bedroom pop\",\n",
    "    \"lofi\",\n",
    "    \"lo-fi\",\n",
    "    \"chill\",\n",
    "    \"ambient\",\n",
    "    \"synthwave\",\n",
    "    \"phonk\",\n",
    "    \"trap remix\",\n",
    "    \"orchestral\",\n",
    "    \"symphonic\",\n",
    "    \"string version\",\n",
    "    \"piano version\",\n",
    "    \"guitar cover\",\n",
    "    \"drum cover\",\n",
    "    \"bass cover\",\n",
    "    \"behind the scenes\",\n",
    "    \"making of\",\n",
    "    \"audio only\",\n",
    "    \"concept video\",\n",
    "    \"fancam\",\n",
    "    \"practice\",\n",
    "    \"dance practice\",\n",
    "    \"line distribution\",\n",
    "    \"tv performance\",\n",
    "    \"award show\",\n",
    "    \"web exclusive\",\n",
    "    \"home recording\",\n",
    "    \"bedroom recording\",\n",
    "    \"raw take\",\n",
    "    \"one take\",\n",
    "    \"studio session\",\n",
    "    \"jam session\",\n",
    "    \"soundcheck\",\n",
    "    \"backstage\",\n",
    "    \"fan cam\",\n",
    "    \"stage mix\"\n",
    "    \"guitar\",\n",
    "    \"electric guitar\",\n",
    "    \"acoustic guitar\",\n",
    "    \"bass\",\n",
    "    \"bass guitar\",\n",
    "    \"piano\",\n",
    "    \"keyboard\",\n",
    "    \"synth\",\n",
    "    \"synthesizer\",\n",
    "    \"drums\",\n",
    "    \"drum cover\",\n",
    "    \"drum pad\",\n",
    "    \"percussion\",\n",
    "    \"violin\",\n",
    "    \"cello\",\n",
    "    \"strings\",\n",
    "    \"saxophone\",\n",
    "    \"trumpet\",\n",
    "    \"trombone\",\n",
    "    \"flute\",\n",
    "    \"clarinet\",\n",
    "    \"harmonica\",\n",
    "    \"banjo\",\n",
    "    \"mandolin\",\n",
    "    \"ukulele\",\n",
    "    \"harp\",\n",
    "    \"organ\",\n",
    "    \"vocals\",\n",
    "    \"vocal cover\",\n",
    "    \"a cappella\",\n",
    "    \"voice\",\n",
    "    \"beatbox\",\"live\",\n",
    "    \"live performance\",\n",
    "    \"studio session\",\n",
    "    \"slowed\",\n",
    "    \"slowed + reverb\",\n",
    "    \"sped up\",\n",
    "    \"reverb\",\n",
    "    \"nightcore\",\n",
    "    \"remix\",\n",
    "    \"bootleg\",\n",
    "    \"edit\",\n",
    "    \"remastered\",\n",
    "    \"remaster\",\n",
    "    \"cover\",\n",
    "    \"instrumental\",\n",
    "    \"karaoke\",\n",
    "    \"acoustic\",\n",
    "    \"piano version\",\n",
    "    \"orchestral version\",\n",
    "    \"extended\",\n",
    "    \"loop\",\n",
    "    \"8d audio\",\n",
    "    \"binaural\",\n",
    "    \"audiophile\",\n",
    "    \"vinyl rip\",\n",
    "    \"cassette rip\",\n",
    "    \"demo\",\n",
    "    \"unreleased\",\n",
    "    \"official audio\",\n",
    "    \"official video\",\n",
    "    \"music video\",\n",
    "    \"visualizer\",\n",
    "    \"fan made\",\n",
    "    \"fan edit\",\n",
    "    \"bass boosted\",\n",
    "    \"HQ\",\n",
    "    \"HD\",\n",
    "    \"360 video\",\n",
    "    \"surround sound\",\n",
    "    \"live from\",\n",
    "    \"Tiny Desk\",\n",
    "    \"MTV Unplugged\",\n",
    "    \"concert version\",\n",
    "    \"radio edit\",\n",
    "    \"clean version\",\n",
    "    \"explicit\",\n",
    "    \"AMV\",  # Anime Music Video\n",
    "    \"lyric video\",\n",
    "    \"with lyrics\",\n",
    "    \"without vocals\",\n",
    "    \"no vocals\",\n",
    "    \"reaction\",\n",
    "    \"mashup\",\n",
    "    \"medley\",\n",
    "    \"DJ set\",\n",
    "    \"soundcheck\",\n",
    "    \"freestyle\",\n",
    "    \"session\",\n",
    "    \"rehearsal\",\n",
    "    \"snippet\",\n",
    "    \"trailer\",\n",
    "    \"teaser\"\n",
    "]\n",
    "\n",
    "\n",
    "terms_set = set(terms_set)\n",
    "print(len(terms_set))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import re\n",
    "\n",
    "# map from the meta_id to the terms\n",
    "\n",
    "terms_map = {}\n",
    "# Precompile regex patterns for better performance\n",
    "term_patterns = {term: re.compile(r'\\b' + re.escape(term) + r'\\b') for term in terms_set}\n",
    "\n",
    "pbar = tqdm(metas)\n",
    "for meta in pbar:\n",
    "    title_text = meta[\"title\"].lower()  # Convert to lowercase once\n",
    "    # Use precompiled patterns for faster matching\n",
    "    found_terms = [term for term, pattern in term_patterns.items() if pattern.search(title_text)]\n",
    "    if found_terms:\n",
    "        terms_map[meta[\"id\"]] = found_terms\n",
    "        pbar.set_postfix(found_terms=len(terms_map))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import json\n",
    "# save the terms_map\n",
    "with open(\"/home/christian/code/christian/metadata/dicsogs_title_terms_map.json\", \"w\") as f:\n",
    "    json.dump(terms_map, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count the the number of terms for each meta_id\n",
    "term_counts = {}\n",
    "for meta_id, terms in terms_map.items():\n",
    "    term_counts[meta_id] = len(terms)\n",
    "\n",
    "# sort the term_counts by value\n",
    "term_counts = sorted(term_counts.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "print(term_counts[1000:1010])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "terms_map[\"Y3gTLXpXxtk\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"/home/christian/code/christian/metadata/terms/discogs_title_terms_map.json\", \"r\") as f:\n",
    "    terms_map = json.load(f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map from the meta_id to the original metas\n",
    "metas_map = {meta[\"id\"]: meta for meta in metas}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count the number of terms in the terms_map\n",
    "print(len(terms_map))\n",
    "print()\n",
    "# count the occurrences of each term\n",
    "term_counts = {}\n",
    "for terms in terms_map.values():\n",
    "    for term in terms:\n",
    "        term_counts[term] = term_counts.get(term, 0) + 1\n",
    "\n",
    "for term, count in sorted(term_counts.items(), key=lambda x: x[1], reverse=True):\n",
    "    print(f\"{count:02d}: {term}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a list of ids for each term    \n",
    "term_ids = {}\n",
    "for term, count in sorted(term_counts.items(), key=lambda x: x[1], reverse=True):\n",
    "    term_ids[term] = []\n",
    "    for meta_id, terms in terms_map.items():\n",
    "        if term in terms:\n",
    "            term_ids[term].append(meta_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 9\n",
    "term = \"binaural\"\n",
    "print(term_ids[term][idx])\n",
    "print(len(term_ids[term]))\n",
    "print(metas_map[term_ids[term][idx]][\"keywords\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}