{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import s3fs\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fs = s3fs.S3FileSystem()\n",
    "\n",
    "def read_ult_guitar(filepath, source='local'):\n",
    "    full_song_json_data = []\n",
    "    if source == 's3':\n",
    "        open_file = fs.open\n",
    "    elif source == 'local':\n",
    "        open_file = open\n",
    "    else:\n",
    "        print('File source not yet supported.')\n",
    "    with open_file(filepath, 'rb') as f:\n",
    "        for line in tqdm(f):\n",
    "            json_line = line.decode('utf-8').strip()\n",
    "            try:\n",
    "                merged_json = {**json.loads(json_line)['tab'], **json.loads(json_line)['tab_view']}\n",
    "                full_song_json_data.append(merged_json)\n",
    "            except json.JSONDecodeError as e:\n",
    "                print(f\"Error parsing JSON in line: {json_line}, Error: {str(e)}\")\n",
    "    return full_song_json_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ult_guitar = read_ult_guitar('tabs.jsonl')\n",
    "ult_guitar_df = pd.DataFrame(ult_guitar)\n",
    "print(len(ult_guitar_df))\n",
    "ult_guitar_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_views(entry):\n",
    "    views = entry['ctx']['tab_views']\n",
    "    return views\n",
    "\n",
    "def get_favs(entry):\n",
    "    favs = entry['ctx']['tab_favourites_count']\n",
    "    return favs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_ult_guitar_df = ult_guitar_df.copy()\n",
    "full_ult_guitar_df['tab_views'] = full_ult_guitar_df['tracking'].apply(get_views)\n",
    "full_ult_guitar_df['tab_favorites'] = full_ult_guitar_df['tracking'].apply(get_favs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_ult_guitar_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_ult_guitar_df['type'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chords_ult_guitar = full_ult_guitar_df[full_ult_guitar_df['type']=='Chords'].reset_index(drop=True)\n",
    "print(len(chords_ult_guitar))\n",
    "chords_ult_guitar.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chords_ult_guitar[chords_ult_guitar.duplicated('song_id', keep=False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chords_ult_guitar_sorted = chords_ult_guitar.sort_values(['song_id', 'tab_favorites'], ascending=[True, False])\n",
    "chords_ult_guitar_songs = chords_ult_guitar_sorted.drop_duplicates(subset='song_id', keep='first').reset_index(drop=True)\n",
    "print(len(chords_ult_guitar_songs))\n",
    "chords_ult_guitar_songs.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove / add columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_content(entry):\n",
    "    content = entry['content']\n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_chords_ult_guitar = chords_ult_guitar_songs.copy()\n",
    "full_chords_ult_guitar['content'] = full_chords_ult_guitar['wiki_tab'].apply(get_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_chords_ult_guitar = full_chords_ult_guitar.drop(columns=['adsupp_binary_blocked', 'official_backingtrack', 'official_backingtrack_url', 'official_tab_url', 'official_tab_want', 'alert', 'pro_tab_onbording', 'piano_inversions', 'new_ug_difficulty', 'brothers_by_type', 'brothers_count', 'binary_id']) # drop NaNs\n",
    "full_chords_ult_guitar = full_chords_ult_guitar.drop(columns=['status', 'tab_access_type', 'tp_version', 'has_preset', 'can_edit_tab', 'can_edit_strumming', 'can_send_tab_stats', 'lyrics_lines']) # drop columns with only one unique value\n",
    "full_chords_ult_guitar = full_chords_ult_guitar.drop(columns='votes') # strictly less info than count_rating\n",
    "full_chords_ult_guitar = full_chords_ult_guitar.drop(columns=['recording', 'album_cover', 'artist_cover', 'best_pro_tab_url', 'has_hq_description', 'has_official_version', 'has_chord_pro', 'song_image', 'last_comments', 'comments', 'stats']) # other misc things we don't need (i.e. image urls, pro-related things)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_chords_ult_guitar.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_chords_ult_guitar = full_chords_ult_guitar.sort_values(by='tab_views', ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "enough_views = sorted_chords_ult_guitar[sorted_chords_ult_guitar['tab_views'] >= 100]\n",
    "enough_favorites = enough_views[enough_views['tab_favorites'] >= 10]\n",
    "remove_albums = enough_favorites[enough_favorites['part'] != 'album']\n",
    "remove_blocked = remove_albums[remove_albums['blocked'] == False]\n",
    "filtered_chords_ult_guitar = remove_blocked[(remove_blocked['count_rating'] < 3) | (remove_blocked['rating'] >= 3.5)]\n",
    "print(len(filtered_chords_ult_guitar))\n",
    "filtered_chords_ult_guitar[['song_name', 'artist_name', 'difficulty', 'rating', 'count_rating', 'tab_views', 'tab_favorites', 'tab_url']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_chords_ult_guitar = filtered_chords_ult_guitar.drop(columns=['type', 'part', 'version_description', 'verified', 'artist_url', 'type_name', 'wiki_tab',\n",
    "                                                                      'versions', 'chord_type', 'tab_corrects', 'is_simplify_available', 'tab_search_link',\n",
    "                                                                      'encode_strummings', 'tab_manually_highlighted', 'smart_auto_scroll_available', 'video_feed',\n",
    "                                                                      'difficulty', 'tracking', 'date', 'date_update', 'preset_id', 'user_id', 'user_iq', 'username',\n",
    "                                                                      'contributors', 'strummings', 'blocked', 'wiki_tab_user_group_id', 'ug_difficulty',\n",
    "                                                                      'wiki_tab_user_iq', 'user_group_id', ]) # throwing away more garbage\n",
    "filtered_chords_ult_guitar.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_chords_ult_guitar = filtered_chords_ult_guitar[['id', 'song_id', 'song_name', 'artist_id', 'artist_name', 'version', 'content', 'tonality_name', 'tab_url', 'rating', 'count_rating', 'tab_views', 'tab_favorites', 'meta']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "section_headers = [\n",
    " 'Intro(?: ?\\d*)?', 'Verse(?: ?\\d*)?', 'Bridge(?: ?\\d*)?', 'Pre-Chorus(?: ?\\d*)?', 'Chorus(?: ?\\d*)?', 'Post-Chorus(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Interlude(?: ?\\d*)?', 'Instrumental(?: ?\\d*)?', 'Outro(?: ?\\d*)?',\n",
    " 'Introducción(?: ?\\d*)?', 'Verso(?: ?\\d*)?', 'Puente(?: ?\\d*)?', 'Pre-Coro(?: ?\\d*)?', 'Coro(?: ?\\d*)?', 'Post-Coro(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Interludio(?: ?\\d*)?', 'Instrumental(?: ?\\d*)?', 'Final(?: ?\\d*)?',\n",
    " '引子(?: ?\\d*)?', '主歌(?: ?\\d*)?', '副歌(?: ?\\d*)?', '前副歌(?: ?\\d*)?', '副歌(?: ?\\d*)?', '后副歌(?: ?\\d*)?', '独奏(?: ?\\d*)?', '插曲(?: ?\\d*)?', '器乐(?: ?\\d*)?', '尾声(?: ?\\d*)?',\n",
    " 'परिचय(?: ?\\d*)?', 'अंतरा(?: ?\\d*)?', 'पुल(?: ?\\d*)?', 'पूर्व-कोरस(?: ?\\d*)?', 'कोरस(?: ?\\d*)?', 'पोस्ट-कोरस(?: ?\\d*)?', 'एकल(?: ?\\d*)?', 'मध्यांतर(?: ?\\d*)?', 'वाद्य संगीत(?: ?\\d*)?', 'समाप्ति(?: ?\\d*)?',\n",
    " 'مقدمة(?: ?\\d*)?', 'مقطع(?: ?\\d*)?', 'جسر(?: ?\\d*)?', 'ما قبل الكورس(?: ?\\d*)?', 'كورس(?: ?\\d*)?', 'ما بعد الكورس(?: ?\\d*)?', 'منفرد(?: ?\\d*)?', 'استراحة(?: ?\\d*)?', 'موسيقى تصويرية(?: ?\\d*)?', 'خاتمة(?: ?\\d*)?',\n",
    " 'পরিচয়(?: ?\\d*)?', 'স্তবক(?: ?\\d*)?', 'সেতু(?: ?\\d*)?', 'প্রাক-গান(?: ?\\d*)?', 'গীত(?: ?\\d*)?', 'গীত-পরবর্তী(?: ?\\d*)?', 'একক(?: ?\\d*)?', 'অন্তরাল(?: ?\\d*)?', 'বাদ্যযন্ত্র(?: ?\\d*)?', 'সমাপ্তি(?: ?\\d*)?',\n",
    " 'Introdução(?: ?\\d*)?', 'Verso(?: ?\\d*)?', 'Ponte(?: ?\\d*)?', 'Pré-Refrão(?: ?\\d*)?', 'Refrão(?: ?\\d*)?', 'Pós-Refrão(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Interlúdio(?: ?\\d*)?', 'Instrumental(?: ?\\d*)?', 'Final(?: ?\\d*)?',\n",
    " 'Введение(?: ?\\d*)?', 'Куплет(?: ?\\d*)?', 'Мост(?: ?\\d*)?', 'Пред-припев(?: ?\\d*)?', 'Припев(?: ?\\d*)?', 'Пост-припев(?: ?\\d*)?', 'Соло(?: ?\\d*)?', 'Интерлюдия(?: ?\\d*)?', 'Инструментал(?: ?\\d*)?', 'Конец(?: ?\\d*)?',\n",
    " 'イントロ(?: ?\\d*)?', '詩(?: ?\\d*)?', 'ブリッジ(?: ?\\d*)?', 'プレ-コーラス(?: ?\\d*)?', 'コーラス(?: ?\\d*)?', 'ポスト-コーラス(?: ?\\d*)?', 'ソロ(?: ?\\d*)?', '間奏(?: ?\\d*)?', 'インストゥルメンタル(?: ?\\d*)?', 'アウトロ(?: ?\\d*)?',\n",
    " 'Einführung(?: ?\\d*)?', 'Strophe(?: ?\\d*)?', 'Brücke(?: ?\\d*)?', 'Vor-Refrain(?: ?\\d*)?', 'Refrain(?: ?\\d*)?', 'Nach-Refrain(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Zwischenteil(?: ?\\d*)?', 'Instrumental(?: ?\\d*)?', 'Outro(?: ?\\d*)?',\n",
    " 'Introduction(?: ?\\d*)?', 'Couplets?(?: ?\\d*)?', 'Pont(?: ?\\d*)?', 'Pré-Refrain(?: ?\\d*)?', 'Refrain(?: ?\\d*)?', 'Post-Refrain(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Interlude(?: ?\\d*)?', 'Instrumental(?: ?\\d*)?', 'Fin(?: ?\\d*)?',\n",
    " 'Introduzione(?: ?\\d*)?', 'Strofa(?: ?\\d*)?', 'Ponte(?: ?\\d*)?', 'Pre-Ritornello(?: ?\\d*)?', 'Ritornello(?: ?\\d*)?', 'Post-Ritornello(?: ?\\d*)?', 'Assolo(?: ?\\d*)?', 'Interludio(?: ?\\d*)?', 'Strumentale(?: ?\\d*)?', 'Finale(?: ?\\d*)?',\n",
    " '도입부(?: ?\\d*)?', '구절(?: ?\\d*)?', '브리지(?: ?\\d*)?', '프리-코러스(?: ?\\d*)?', '코러스(?: ?\\d*)?', '포스트-코러스(?: ?\\d*)?', '솔로(?: ?\\d*)?', '간주(?: ?\\d*)?', '기악곡(?: ?\\d*)?', '아웃트로(?: ?\\d*)?',\n",
    " 'Giriş(?: ?\\d*)?', 'Kıta(?: ?\\d*)?', 'Köprü(?: ?\\d*)?', 'Ön-Nakarat(?: ?\\d*)?', 'Nakarat(?: ?\\d*)?', 'Nakarat Sonrası(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Ara Bölüm(?: ?\\d*)?', 'Enstrümantal(?: ?\\d*)?', 'Çıkış(?: ?\\d*)?',\n",
    " 'Giới thiệu(?: ?\\d*)?', 'Điệp khúc(?: ?\\d*)?', 'Cầu nối(?: ?\\d*)?', 'Trước điệp khúc(?: ?\\d*)?', 'Điệp khúc(?: ?\\d*)?', 'Sau điệp khúc(?: ?\\d*)?', 'Độc tấu(?: ?\\d*)?', 'Đoạn nhạc phụ(?: ?\\d*)?', 'Nhạc cụ(?: ?\\d*)?', 'Kết thúc(?: ?\\d*)?',\n",
    " 'Inledning(?: ?\\d*)?', 'Vers(?: ?\\d*)?', 'Brygga(?: ?\\d*)?', 'Före-refräng(?: ?\\d*)?', 'Refräng(?: ?\\d*)?', 'Efter-refräng(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Mellanspel(?: ?\\d*)?', 'Instrumental(?: ?\\d*)?', 'Avslutning(?: ?\\d*)?',\n",
    " 'Intro(?: ?\\d*)?', 'Zwrotka(?: ?\\d*)?', 'Most(?: ?\\d*)?', 'Przed-refren(?: ?\\d*)?', 'Refren(?: ?\\d*)?', 'Po-refren(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Interludium(?: ?\\d*)?', 'Instrumentalny(?: ?\\d*)?', 'Outro(?: ?\\d*)?',\n",
    " 'Johdanto(?: ?\\d*)?', 'Säkeistö(?: ?\\d*)?', 'Silta(?: ?\\d*)?', 'Esi-kertosäe(?: ?\\d*)?', 'Kertosäe(?: ?\\d*)?', 'Jälki-kertosäe(?: ?\\d*)?', 'Soolo(?: ?\\d*)?', 'Välisoitto(?: ?\\d*)?', 'Instrumentaali(?: ?\\d*)?', 'Loppusoitto(?: ?\\d*)?',\n",
    " 'Innledning(?: ?\\d*)?', 'Vers(?: ?\\d*)?', 'Bro(?: ?\\d*)?', 'Pre-refreng(?: ?\\d*)?', 'Refreng(?: ?\\d*)?', 'Post-refreng(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Mellomspill(?: ?\\d*)?', 'Instrumental(?: ?\\d*)?', 'Avslutning(?: ?\\d*)?',\n",
    " 'Inleiding(?: ?\\d*)?', 'Couplet(?: ?\\d*)?', 'Brug(?: ?\\d*)?', 'Pre-refrein(?: ?\\d*)?', 'Refrein(?: ?\\d*)?', 'Post-refrein(?: ?\\d*)?', 'Solo(?: ?\\d*)?', 'Tussenspel(?: ?\\d*)?', 'Instrumentaal(?: ?\\d*)?', 'Outro(?: ?\\d*)?',\n",
    " 'Εισαγωγή(?: ?\\d*)?', 'Στροφή(?: ?\\d*)?', 'Γέφυρα(?: ?\\d*)?', 'Προ-ρεφρέν(?: ?\\d*)?', 'Ρεφρέν(?: ?\\d*)?', 'Μετά-ρεφρέν(?: ?\\d*)?', 'Σόλο(?: ?\\d*)?', 'Ιντερλούδιο(?: ?\\d*)?', 'Ορχηστρικό(?: ?\\d*)?', 'Έξοδος(?: ?\\d*)?'\n",
    "]\n",
    "\n",
    "def remove_parentheses(match):\n",
    "    return f'[ch]{match.group(1)}[/ch]'\n",
    "\n",
    "def parse_chords_lyrics_exact(input_string):\n",
    "    ### some base cleaning\n",
    "    cleaned_string = re.sub(r'\\(\\[ch\\](.*?)\\[/ch\\]\\)', remove_parentheses, input_string)\n",
    "    section_pattern = r'(?i)\\[(' + '|'.join(section_headers) + r')\\]' \n",
    "\n",
    "    sections = re.split(section_pattern, cleaned_string)[1:] # find sections\n",
    "    headers = re.findall(section_pattern, cleaned_string)\n",
    "\n",
    "    parsed_lines = []\n",
    "    for i in range(len(headers)): # if the tab has headers\n",
    "        header = headers[i]\n",
    "        content = sections[i * 2 + 1].strip()  # content every second element after header\n",
    "        parsed_lines.append(f'[{header}]')\n",
    "\n",
    "        pattern = r'\\[tab\\](.*?)\\[/tab\\]'\n",
    "        chord_lyric_lines = re.findall(pattern, content, re.DOTALL)\n",
    "\n",
    "        if not chord_lyric_lines: # no matches found, i.e. no chord-lyric [tab]\n",
    "            chords = []\n",
    "            for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', content):\n",
    "                chords.append(f'[{match.group(1).strip()}]')\n",
    "            instrumental_line = ' '.join(chords)\n",
    "            parsed_lines.append(instrumental_line)\n",
    "\n",
    "        for line in chord_lyric_lines: # go through [tab] instances, parse chords, embed them into lyrics\n",
    "            if '----' not in line:\n",
    "                lyrics = re.sub(r'\\[ch\\](.*?)\\[/ch\\]', '', line, flags=re.DOTALL).strip()\n",
    "                previous_end = 0\n",
    "                space_counts = []\n",
    "                chords = []\n",
    "\n",
    "                for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', line):\n",
    "                    chords.append(match.group(1).strip())\n",
    "                    start = match.start()  \n",
    "                    num_spaces_between = start - previous_end  \n",
    "                    space_counts.append(num_spaces_between)\n",
    "                    previous_end = match.end()\n",
    "                locations = np.cumsum(np.array(space_counts))\n",
    "\n",
    "                lyric_list = list(lyrics)\n",
    "                extra_chars = 0\n",
    "                for i in range(len(chords)):\n",
    "                    lyric_list.insert(locations[i] + extra_chars, f'[{chords[i]}]')\n",
    "                    extra_chars += len([{chords[i]}]) + 2 # extra spacing b.c. UI\n",
    "                lyrics_w_chords = ''.join(lyric_list)\n",
    "                parsed_lines.append(lyrics_w_chords)\n",
    "        parsed_lines.append('\\n')\n",
    "    if not parsed_lines: # no recognized section headers, just add the tabs\n",
    "        pattern = r'\\[tab\\](.*?)\\[/tab\\]'\n",
    "        chord_lyric_lines = re.findall(pattern, cleaned_string, re.DOTALL)\n",
    "        non_tabs = re.sub(r'\\[tab\\](.*?)\\[/tab\\]', '', cleaned_string, flags=re.DOTALL).strip()\n",
    "        non_chords_tabs = re.sub(r'\\[ch\\](.*?)\\[/ch\\]', '', non_tabs, flags=re.DOTALL).strip()\n",
    "        \n",
    "        # don't add floating instrumentals if not contained in a section, since there would be no reference?\n",
    "        if not chord_lyric_lines: # no matches found, i.e. no chord-lyric [tab]\n",
    "            chords = []\n",
    "            for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', cleaned_string):\n",
    "                chords.append(f'[{match.group(1).strip()}]')\n",
    "            instrumental_line = ' '.join(chords)\n",
    "            parsed_lines.append(instrumental_line)\n",
    "\n",
    "        for line in chord_lyric_lines: # go through [tab] instances, parse chords, embed them into lyrics\n",
    "            if '----' not in line:\n",
    "                lyrics = re.sub(r'\\[ch\\](.*?)\\[/ch\\]', '', line).strip()\n",
    "                previous_end = 0\n",
    "                space_counts = []\n",
    "                chords = []\n",
    "\n",
    "                for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', line):\n",
    "                    chords.append(match.group(1).strip())\n",
    "                    start = match.start()  \n",
    "                    num_spaces_between = start - previous_end  \n",
    "                    space_counts.append(num_spaces_between)\n",
    "                    previous_end = match.end()\n",
    "                locations = np.cumsum(np.array(space_counts))\n",
    "\n",
    "                lyric_list = list(lyrics)\n",
    "                extra_chars = 0\n",
    "                for i in range(len(chords)):\n",
    "                    lyric_list.insert(locations[i] + extra_chars, f'[{chords[i]}]')\n",
    "                    extra_chars += len([{chords[i]}]) + 2 # extra spacing b.c. UI\n",
    "                lyrics_w_chords = ''.join(lyric_list)\n",
    "                parsed_lines.append(lyrics_w_chords)\n",
    "        if len(non_chords_tabs) > 0.2*len(' '.join(chord_lyric_lines)): # indicator of lots of junk, we'll just filter these out\n",
    "            parsed_lines = []\n",
    "        if re.findall(r'(?i)Repeat', non_chords_tabs) or re.findall(r'(?i)\\(x\\d*\\)', non_chords_tabs): # repeats are messed up, filter them out\n",
    "            parsed_lines = []\n",
    "        parsed_lines.append('\\n')\n",
    "        \n",
    "    return '\\n'.join(parsed_lines)\n",
    "\n",
    "def parse_chords_lyrics_approx(input_string):\n",
    "    ### some base cleaning\n",
    "    cleaned_string = re.sub(r'\\(\\[ch\\](.*?)\\[/ch\\]\\)', remove_parentheses, input_string)\n",
    "    section_pattern = r'(?i)\\[(' + '|'.join(section_headers) + r')\\]' \n",
    "\n",
    "    sections = re.split(section_pattern, cleaned_string)[1:] # find sections\n",
    "    headers = re.findall(section_pattern, cleaned_string)\n",
    "\n",
    "    parsed_lines = []\n",
    "    for i in range(len(headers)): # if the tab has headers\n",
    "        header = headers[i]\n",
    "        content = sections[i * 2 + 1].strip()  # content every second element after header\n",
    "        parsed_lines.append(f'[{header}]')\n",
    "\n",
    "        pattern = r'\\[tab\\](.*?)\\[/tab\\]'\n",
    "        chord_lyric_lines = re.findall(pattern, content, re.DOTALL)\n",
    "\n",
    "        if not chord_lyric_lines: # no matches found, i.e. no chord-lyric [tab]\n",
    "            chords = []\n",
    "            for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', content):\n",
    "                chords.append(f'[{match.group(1).strip()}]')\n",
    "            instrumental_line = ' '.join(chords)\n",
    "            parsed_lines.append(instrumental_line)\n",
    "\n",
    "        for line in chord_lyric_lines:\n",
    "            if '----' not in line:\n",
    "                lyrics = re.sub(r'\\[ch\\](.*?)\\[/ch\\]', '', line, flags=re.DOTALL).strip()\n",
    "                chords = [match.group(1).strip() for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', line)]\n",
    "                chord_positions = [match.start() for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', line)]\n",
    "\n",
    "                # Split lyrics into words\n",
    "                words = lyrics.split()\n",
    "                \n",
    "                # Handle potential divide-by-zero cases\n",
    "                lyrics_length = len(lyrics)\n",
    "                max_chord_position = max(chord_positions) if chord_positions else 0\n",
    "                \n",
    "                if lyrics_length == 0 and max_chord_position == 0:\n",
    "                    # Both lyrics and chords are empty, nothing to do\n",
    "                    continue\n",
    "                elif lyrics_length == 0 and max_chord_position > 0:\n",
    "                    # Lyrics are empty but chords exist, place all chords at the beginning\n",
    "                    parsed_lines.append(' '.join(f'[{chord}]' for chord in chords))\n",
    "                    continue\n",
    "                elif max_chord_position == 0 and lyrics_length > 0:\n",
    "                    # No chords, just add lyrics\n",
    "                    parsed_lines.append(lyrics)\n",
    "                    continue\n",
    "                \n",
    "                # Normal case: both lyrics and chords exist\n",
    "                scale_factor = lyrics_length / max(max_chord_position, lyrics_length)\n",
    "                scaled_chord_positions = [int(pos * scale_factor) for pos in chord_positions]\n",
    "\n",
    "                # Assign chords to words\n",
    "                word_chord_map = {i: [] for i in range(len(words))}\n",
    "                current_word_index = 0\n",
    "                current_word_start = 0\n",
    "\n",
    "                for chord, position in zip(chords, scaled_chord_positions):\n",
    "                    while current_word_index < len(words) and current_word_start + len(words[current_word_index]) <= position:\n",
    "                        current_word_start += len(words[current_word_index]) + 1  # +1 for space\n",
    "                        current_word_index += 1\n",
    "                    if current_word_index < len(words):\n",
    "                        word_chord_map[current_word_index].append(chord)\n",
    "\n",
    "                # Construct result\n",
    "                result = []\n",
    "                for i, word in enumerate(words):\n",
    "                    chords_for_word = word_chord_map[i]\n",
    "                    if chords_for_word:\n",
    "                        result.extend(f'[{chord}]' for chord in chords_for_word)\n",
    "                    result.append(word)\n",
    "\n",
    "                lyrics_w_chords = ' '.join(result)\n",
    "                parsed_lines.append(lyrics_w_chords)\n",
    "        parsed_lines.append('\\n')\n",
    "    if not parsed_lines: # no recognized section headers, just add the tabs\n",
    "        pattern = r'\\[tab\\](.*?)\\[/tab\\]'\n",
    "        chord_lyric_lines = re.findall(pattern, cleaned_string, re.DOTALL)\n",
    "        non_tabs = re.sub(r'\\[tab\\](.*?)\\[/tab\\]', '', cleaned_string, flags=re.DOTALL).strip()\n",
    "        non_chords_tabs = re.sub(r'\\[ch\\](.*?)\\[/ch\\]', '', non_tabs, flags=re.DOTALL).strip()\n",
    "        \n",
    "        # don't add floating instrumentals if not contained in a section, since there would be no reference?\n",
    "        if not chord_lyric_lines: # no matches found, i.e. no chord-lyric [tab]\n",
    "            chords = []\n",
    "            for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', cleaned_string):\n",
    "                chords.append(f'[{match.group(1).strip()}]')\n",
    "            instrumental_line = ' '.join(chords)\n",
    "            parsed_lines.append(instrumental_line)\n",
    "\n",
    "        for line in chord_lyric_lines:\n",
    "            if '----' not in line:\n",
    "                lyrics = re.sub(r'\\[ch\\](.*?)\\[/ch\\]', '', line, flags=re.DOTALL).strip()\n",
    "                chords = [match.group(1).strip() for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', line)]\n",
    "                chord_positions = [match.start() for match in re.finditer(r'\\[ch\\](.*?)\\[/ch\\]', line)]\n",
    "\n",
    "                # Split lyrics into words\n",
    "                words = lyrics.split()\n",
    "                \n",
    "                # Handle potential divide-by-zero cases\n",
    "                lyrics_length = len(lyrics)\n",
    "                max_chord_position = max(chord_positions) if chord_positions else 0\n",
    "                \n",
    "                if lyrics_length == 0 and max_chord_position == 0:\n",
    "                    # Both lyrics and chords are empty, nothing to do\n",
    "                    continue\n",
    "                elif lyrics_length == 0 and max_chord_position > 0:\n",
    "                    # Lyrics are empty but chords exist, place all chords at the beginning\n",
    "                    parsed_lines.append(' '.join(f'[{chord}]' for chord in chords))\n",
    "                    continue\n",
    "                elif max_chord_position == 0 and lyrics_length > 0:\n",
    "                    # No chords, just add lyrics\n",
    "                    parsed_lines.append(lyrics)\n",
    "                    continue\n",
    "                \n",
    "                # Normal case: both lyrics and chords exist\n",
    "                scale_factor = lyrics_length / max(max_chord_position, lyrics_length)\n",
    "                scaled_chord_positions = [int(pos * scale_factor) for pos in chord_positions]\n",
    "\n",
    "                # Assign chords to words\n",
    "                word_chord_map = {i: [] for i in range(len(words))}\n",
    "                current_word_index = 0\n",
    "                current_word_start = 0\n",
    "\n",
    "                for chord, position in zip(chords, scaled_chord_positions):\n",
    "                    while current_word_index < len(words) and current_word_start + len(words[current_word_index]) <= position:\n",
    "                        current_word_start += len(words[current_word_index]) + 1  # +1 for space\n",
    "                        current_word_index += 1\n",
    "                    if current_word_index < len(words):\n",
    "                        word_chord_map[current_word_index].append(chord)\n",
    "\n",
    "                # Construct result\n",
    "                result = []\n",
    "                for i, word in enumerate(words):\n",
    "                    chords_for_word = word_chord_map[i]\n",
    "                    if chords_for_word:\n",
    "                        result.extend(f'[{chord}]' for chord in chords_for_word)\n",
    "                    result.append(word)\n",
    "\n",
    "                lyrics_w_chords = ' '.join(result)\n",
    "                parsed_lines.append(lyrics_w_chords)\n",
    "                \n",
    "        if len(non_chords_tabs) > 0.2*len(' '.join(chord_lyric_lines)): # indicator of lots of junk, we'll just filter these out\n",
    "            parsed_lines = []\n",
    "        if re.findall(r'(?i)Repeat', non_chords_tabs) or re.findall(r'(?i)\\(x\\d*\\)', non_chords_tabs): # repeats are messed up, filter them out\n",
    "            parsed_lines = []\n",
    "        parsed_lines.append('\\n')\n",
    "        \n",
    "    return '\\n'.join(parsed_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_chords_ult_guitar_df = filtered_chords_ult_guitar.copy()\n",
    "cleaned_chords_ult_guitar_df['aligned_chords_lyrics'] = filtered_chords_ult_guitar['content'].apply(parse_chords_lyrics_exact)\n",
    "cleaned_chords_ult_guitar_df['aligned_chords_lyrics_approx'] = filtered_chords_ult_guitar['content'].apply(parse_chords_lyrics_approx)\n",
    "cleaned_chords_ult_guitar_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_chords_ult_guitar_df = cleaned_chords_ult_guitar_df.drop(columns='content')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(cleaned_chords_ult_guitar_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_cleaned_chords_ult_guitar_df = cleaned_chords_ult_guitar_df[cleaned_chords_ult_guitar_df['aligned_chords_lyrics'].str.strip().apply(len) != 0]\n",
    "print(len(final_cleaned_chords_ult_guitar_df))\n",
    "final_cleaned_chords_ult_guitar_json_str = final_cleaned_chords_ult_guitar_df.to_json(orient='records', lines=True)\n",
    "final_cleaned_chords_ult_guitar_json_str = final_cleaned_chords_ult_guitar_json_str.replace('\\\\/', '/')\n",
    "with open('cleaned_ult_guitar.jsonl', 'w') as f:\n",
    "    f.write(final_cleaned_chords_ult_guitar_json_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sample = filtered_chords_ult_guitar.sample(300)\n",
    "random_sample.to_csv('random_sample_tabs_test.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}