{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from suno_utils.utils.text import read_jsonl, write_jsonl\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "beatport_mashups = read_jsonl(\"mashup_beatport_match.jsonl\")\n",
    "traxsource_mashups = read_jsonl(\"mashup_traxsource_match.jsonl\")\n",
    "rym_mashups = read_jsonl(\"mashup_rym_match.jsonl\")\n",
    "discogs_mashups = read_jsonl(\"mashup_discogs_match.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Beatport: {len(beatport_mashups)}\")\n",
    "print(f\"Traxsource: {len(traxsource_mashups)}\")\n",
    "print(f\"Rate Your Music: {len(rym_mashups)}\")\n",
    "print(f\"Discogs: {len(discogs_mashups)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_potential_mashups = []\n",
    "\n",
    "for data in beatport_mashups:\n",
    "    meta = data['original_entry']\n",
    "    scores = data['scores']\n",
    "    metadata = dict(\n",
    "        song_name=meta['title'],\n",
    "        artists=meta['artist'],\n",
    "        album_name=None,\n",
    "        genre=meta['genre'],\n",
    "        key=meta['key'],\n",
    "        bpm=meta['bpm'],\n",
    "        label=meta['label'],\n",
    "        release_date=meta['release_date'],\n",
    "        delimiter_found=scores['delimiter_found'],\n",
    "        mashup_keyword=scores['mashup_keyword'],\n",
    "        known_mashup_source=scores['known_mashup_source'],\n",
    "        data_source='beatport'\n",
    "    )\n",
    "    all_potential_mashups.append(metadata)\n",
    "\n",
    "for data in traxsource_mashups:\n",
    "    meta = data['original_entry']\n",
    "    scores = data['scores']\n",
    "    metadata = dict(\n",
    "        song_name=meta['song_name'] + \" (\" + meta['version'] +')',\n",
    "        artists=meta['artist_names'],\n",
    "        album_name=meta['release_name'],\n",
    "        genre=meta['genre'],\n",
    "        key=meta['key'],\n",
    "        bpm=meta['bpm'],\n",
    "        label=meta['label'],\n",
    "        release_date=meta['released_date'],\n",
    "        delimiter_found=scores['delimiter_found'],\n",
    "        mashup_keyword=scores['mashup_keyword'],\n",
    "        known_mashup_source=scores['known_mashup_source'],\n",
    "        data_source='traxsource'\n",
    "    )\n",
    "    all_potential_mashups.append(metadata)\n",
    "\n",
    "for data in rym_mashups:\n",
    "    meta = data['original_entry']\n",
    "    scores = data['scores']\n",
    "    metadata = dict(\n",
    "        song_name=meta['title'],\n",
    "        artists=[meta['artist']],\n",
    "        album_name=meta['album'],\n",
    "        genre=meta['genre'],\n",
    "        key=None,\n",
    "        bpm=None,\n",
    "        label=None,\n",
    "        release_date=None,\n",
    "        delimiter_found=scores['delimiter_found'],\n",
    "        mashup_keyword=scores['mashup_keyword'],\n",
    "        known_mashup_source=scores['known_mashup_source'],\n",
    "        data_source='rym'\n",
    "    )\n",
    "    all_potential_mashups.append(metadata)\n",
    "\n",
    "\n",
    "for data in discogs_mashups:\n",
    "    meta = data['original_entry']\n",
    "    scores = data['scores']\n",
    "    metadata = dict(\n",
    "        song_name=meta['title'],\n",
    "        artists=[meta['artist']] if 'artist' in meta else None,\n",
    "        album_name=meta['album'],\n",
    "        genre=meta['genre'] if 'genre' in meta else None,\n",
    "        key=None,\n",
    "        bpm=None,\n",
    "        label=None,\n",
    "        release_date=None,\n",
    "        delimiter_found=scores['delimiter_found'],\n",
    "        mashup_keyword=scores['mashup_keyword'],\n",
    "        known_mashup_source=scores['known_mashup_source'],\n",
    "        data_source='discogs'\n",
    "    )\n",
    "    all_potential_mashups.append(metadata)\n",
    "\n",
    "print(len(all_potential_mashups))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "write_jsonl(all_potential_mashups, \"/home/sara/task_data/consolidated_mashup_data.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "mashup_df = pd.DataFrame(all_potential_mashups)\n",
    "mashup_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all non-None values for the three columns\n",
    "delimiter_found_values = mashup_df[mashup_df['delimiter_found'].notna()]['delimiter_found'].unique()\n",
    "mashup_keyword_values = mashup_df[mashup_df['mashup_keyword'].notna()]['mashup_keyword'].unique()\n",
    "known_mashup_source_values = mashup_df[mashup_df['known_mashup_source'].notna()]['known_mashup_source'].unique()\n",
    "\n",
    "print(\"Unique non-None values in 'delimiter_found':\")\n",
    "print(delimiter_found_values)\n",
    "print(f\"\\nCount: {len(delimiter_found_values)}\")\n",
    "\n",
    "print(\"\\n\" + \"=\"*50)\n",
    "print(\"\\nUnique non-None values in 'mashup_keyword':\")\n",
    "print(mashup_keyword_values)\n",
    "print(f\"\\nCount: {len(mashup_keyword_values)}\")\n",
    "\n",
    "print(\"\\n\" + \"=\"*50)\n",
    "print(\"\\nUnique non-None values in 'known_mashup_source':\")\n",
    "print(known_mashup_source_values)\n",
    "print(f\"\\nCount: {len(known_mashup_source_values)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for cases where both delimiter_found and mashup_keyword are not None\n",
    "filtered_mashups = mashup_df[(mashup_df['delimiter_found'].notna()) & (mashup_df['mashup_keyword'].notna())]\n",
    "\n",
    "print(f\"Total rows with both delimiter_found and mashup_keyword not None: {len(filtered_mashups)}\")\n",
    "print(f\"Percentage of total: {len(filtered_mashups) / len(mashup_df) * 100:.2f}%\")\n",
    "\n",
    "filtered_mashups.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for specific mashup keywords from original dataframe\n",
    "mashup_keywords_of_interest = ['mashup', 'mash-up', 'mash up', 'bootleg']\n",
    "\n",
    "filtered_by_keywords = mashup_df[mashup_df['mashup_keyword'].isin(mashup_keywords_of_interest)]\n",
    "\n",
    "print(f\"Total rows with specified mashup keywords: {len(filtered_by_keywords)}\")\n",
    "print(f\"Percentage of total data: {len(filtered_by_keywords) / len(mashup_df) * 100:.2f}%\")\n",
    "\n",
    "print(\"\\nBreakdown by keyword:\")\n",
    "print(filtered_by_keywords['mashup_keyword'].value_counts())\n",
    "\n",
    "filtered_by_keywords.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for specific delimiter_found values\n",
    "delimiters_of_interest = ['vs', 'x', 'vs.', '//', 'versus']\n",
    "\n",
    "filtered_by_delimiter = mashup_df[mashup_df['delimiter_found'].isin(delimiters_of_interest)]\n",
    "\n",
    "print(f\"Total rows with specified delimiters: {len(filtered_by_delimiter)}\")\n",
    "print(f\"Percentage of total data: {len(filtered_by_delimiter) / len(mashup_df) * 100:.2f}%\")\n",
    "\n",
    "print(\"\\nBreakdown by delimiter:\")\n",
    "print(filtered_by_delimiter['delimiter_found'].value_counts())\n",
    "\n",
    "filtered_by_delimiter.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine all three filters (keep rows that exist in ANY of the three filtered dataframes)\n",
    "# Use index union to avoid issues with unhashable types (lists in artists column)\n",
    "combined_indices = filtered_by_delimiter.index.union(filtered_by_keywords.index).union(filtered_mashups.index)\n",
    "combined_filtered = mashup_df.loc[combined_indices]\n",
    "\n",
    "print(f\"Total rows after combining all filters: {len(combined_filtered)}\")\n",
    "print(f\"Percentage of total data: {len(combined_filtered) / len(mashup_df) * 100:.2f}%\")\n",
    "\n",
    "print(\"\\nBreakdown by data source:\")\n",
    "print(combined_filtered['data_source'].value_counts())\n",
    "\n",
    "print(\"\\nBreakdown by mashup_keyword:\")\n",
    "print(combined_filtered['mashup_keyword'].value_counts())\n",
    "\n",
    "print(\"\\nBreakdown by delimiter_found:\")\n",
    "print(combined_filtered['delimiter_found'].value_counts())\n",
    "\n",
    "combined_filtered.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# Function to check if a position is inside parentheses/brackets\n",
    "def is_inside_brackets(text, pos):\n",
    "    \"\"\"Check if position is inside (), [], or {}\"\"\"\n",
    "    # Count opening and closing brackets before this position\n",
    "    before = text[:pos]\n",
    "    paren_depth = before.count('(') - before.count(')')\n",
    "    square_depth = before.count('[') - before.count(']')\n",
    "    curly_depth = before.count('{') - before.count('}')\n",
    "    \n",
    "    # If any depth is > 0, we're inside brackets\n",
    "    return paren_depth > 0 or square_depth > 0 or curly_depth > 0\n",
    "\n",
    "# Function to split song name by delimiter if it exists\n",
    "def split_by_delimiter(row):\n",
    "    if pd.isna(row['delimiter_found']) or row['delimiter_found'] is None:\n",
    "        return None\n",
    "    \n",
    "    delimiter = row['delimiter_found']\n",
    "    song_name = row['song_name']\n",
    "    \n",
    "    # Create a regex pattern that matches the delimiter surrounded by spaces (case insensitive)\n",
    "    # Escape special regex characters in the delimiter\n",
    "    escaped_delimiter = re.escape(delimiter)\n",
    "    pattern = rf'\\s+{escaped_delimiter}\\s+'\n",
    "    \n",
    "    # Find all matches of the delimiter\n",
    "    matches = list(re.finditer(pattern, song_name, flags=re.IGNORECASE))\n",
    "    \n",
    "    if not matches:\n",
    "        return None\n",
    "    \n",
    "    # Filter out matches that are inside brackets\n",
    "    valid_matches = []\n",
    "    for match in matches:\n",
    "        # Check the middle position of the match\n",
    "        mid_pos = (match.start() + match.end()) // 2\n",
    "        if not is_inside_brackets(song_name, mid_pos):\n",
    "            valid_matches.append(match)\n",
    "    \n",
    "    if not valid_matches:\n",
    "        return None\n",
    "    \n",
    "    # Split by valid delimiters only\n",
    "    parts = []\n",
    "    last_end = 0\n",
    "    for match in valid_matches:\n",
    "        parts.append(song_name[last_end:match.start()].strip())\n",
    "        last_end = match.end()\n",
    "    parts.append(song_name[last_end:].strip())\n",
    "    \n",
    "    # Filter out empty parts\n",
    "    parts = [part for part in parts if part]\n",
    "    \n",
    "    if len(parts) <= 1:\n",
    "        return None\n",
    "    \n",
    "    # Check if any part is completely encapsulated in brackets/parentheses\n",
    "    # Patterns: (text), [text], {text}\n",
    "    for part in parts:\n",
    "        stripped = part.strip()\n",
    "        if ((stripped.startswith('(') and stripped.endswith(')')) or\n",
    "            (stripped.startswith('[') and stripped.endswith(']')) or\n",
    "            (stripped.startswith('{') and stripped.endswith('}'))):\n",
    "            return None\n",
    "    \n",
    "    return parts\n",
    "\n",
    "# Apply the function to create a new column\n",
    "combined_filtered['split_parts'] = combined_filtered.apply(split_by_delimiter, axis=1)\n",
    "\n",
    "# Show some statistics\n",
    "print(f\"Rows with split parts: {combined_filtered['split_parts'].notna().sum()}\")\n",
    "print(f\"Rows without split parts: {combined_filtered['split_parts'].isna().sum()}\")\n",
    "\n",
    "# Show examples\n",
    "print(\"\\nExamples of split results:\")\n",
    "print(combined_filtered[combined_filtered['split_parts'].notna()][['song_name', 'delimiter_found', 'split_parts']].head(10))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for rows where split was successful\n",
    "successfully_split = combined_filtered[combined_filtered['split_parts'].notna()].copy()\n",
    "\n",
    "print(f\"Total rows with successful splits: {len(successfully_split)}\")\n",
    "print(f\"Percentage of combined filtered data: {len(successfully_split) / len(combined_filtered) * 100:.2f}%\")\n",
    "\n",
    "successfully_split.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract metadata from brackets in split_parts\n",
    "def extract_metadata(row):\n",
    "    split_parts = row['split_parts']\n",
    "    \n",
    "    # Check if split_parts is None or empty\n",
    "    if split_parts is None or (isinstance(split_parts, float) and pd.isna(split_parts)):\n",
    "        return None\n",
    "    \n",
    "    all_metadata = []\n",
    "    \n",
    "    for part in split_parts:\n",
    "        # Find all patterns in (), [], {}\n",
    "        patterns = [\n",
    "            r'\\([^)]+\\)',  # (text)\n",
    "            r'\\[[^\\]]+\\]',  # [text]\n",
    "            r'\\{[^}]+\\}'    # {text}\n",
    "        ]\n",
    "        \n",
    "        for pattern in patterns:\n",
    "            matches = re.findall(pattern, part)\n",
    "            for match in matches:\n",
    "                # Remove the brackets and add to metadata\n",
    "                content = match[1:-1].strip()  # Remove first and last char (brackets)\n",
    "                if content:\n",
    "                    all_metadata.append(content)\n",
    "    \n",
    "    # Return comma-separated metadata\n",
    "    return ', '.join(all_metadata) if all_metadata else None\n",
    "\n",
    "# Function to remove metadata from split_parts\n",
    "def clean_split_parts(row):\n",
    "    split_parts = row['split_parts']\n",
    "    \n",
    "    # Check if split_parts is None or empty\n",
    "    if split_parts is None or (isinstance(split_parts, float) and pd.isna(split_parts)):\n",
    "        return split_parts\n",
    "    \n",
    "    cleaned_parts = []\n",
    "    \n",
    "    for part in split_parts:\n",
    "        # Remove all patterns in (), [], {}\n",
    "        patterns = [\n",
    "            r'\\([^)]+\\)',  # (text)\n",
    "            r'\\[[^\\]]+\\]',  # [text]\n",
    "            r'\\{[^}]+\\}'    # {text}\n",
    "        ]\n",
    "        \n",
    "        cleaned_part = part\n",
    "        for pattern in patterns:\n",
    "            cleaned_part = re.sub(pattern, '', cleaned_part)\n",
    "        \n",
    "        # Clean up extra whitespace\n",
    "        cleaned_part = ' '.join(cleaned_part.split()).strip()\n",
    "        \n",
    "        # Only add non-empty parts\n",
    "        if cleaned_part:\n",
    "            cleaned_parts.append(cleaned_part)\n",
    "    \n",
    "    # Return cleaned parts, or None if all parts became empty\n",
    "    return cleaned_parts if cleaned_parts else None\n",
    "\n",
    "# Apply the function to extract metadata\n",
    "successfully_split['metadata'] = successfully_split.apply(extract_metadata, axis=1)\n",
    "\n",
    "# Apply the function to clean split_parts\n",
    "successfully_split['split_parts'] = successfully_split.apply(clean_split_parts, axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "successfully_split.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter 1: More than one artist\n",
    "filter_multiple_artists = successfully_split[successfully_split['artists'].apply(lambda x: x is not None and len(x) > 1)]\n",
    "\n",
    "print(f\"Rows with more than one artist: {len(filter_multiple_artists)}\")\n",
    "print(f\"Percentage of successfully_split: {len(filter_multiple_artists) / len(successfully_split) * 100:.2f}%\")\n",
    "\n",
    "filter_multiple_artists.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter 2: mashup_keyword is not None\n",
    "filter_has_keyword = successfully_split[successfully_split['mashup_keyword'].notna()]\n",
    "\n",
    "print(f\"Rows with mashup_keyword not None: {len(filter_has_keyword)}\")\n",
    "print(f\"Percentage of successfully_split: {len(filter_has_keyword) / len(successfully_split) * 100:.2f}%\")\n",
    "\n",
    "print(\"\\nBreakdown by keyword:\")\n",
    "print(filter_has_keyword['mashup_keyword'].value_counts())\n",
    "\n",
    "filter_has_keyword.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter 3: At least one split part is > 10 characters OR has > 2 words\n",
    "def has_substantial_part(row):\n",
    "    split_parts = row['split_parts']\n",
    "    if split_parts is None or (isinstance(split_parts, float) and pd.isna(split_parts)):\n",
    "        return False\n",
    "    \n",
    "    for part in split_parts:\n",
    "        # Check if part is longer than 10 characters\n",
    "        if len(part) > 10:\n",
    "            return True\n",
    "        # Check if part has more than 2 words\n",
    "        word_count = len(part.split())\n",
    "        if word_count > 2:\n",
    "            return True\n",
    "    \n",
    "    return False\n",
    "\n",
    "filter_substantial_parts = successfully_split[successfully_split.apply(has_substantial_part, axis=1)]\n",
    "\n",
    "print(f\"Rows with at least one substantial part (>10 chars OR >2 words): {len(filter_substantial_parts)}\")\n",
    "print(f\"Percentage of successfully_split: {len(filter_substantial_parts) / len(successfully_split) * 100:.2f}%\")\n",
    "\n",
    "filter_substantial_parts.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_filter = filter_substantial_parts.drop(columns=['known_mashup_source'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_filter.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": [
    "delimiter_found_values = cleaned_filter[cleaned_filter['delimiter_found'].notna()]['delimiter_found'].unique()\n",
    "mashup_keyword_values = cleaned_filter[cleaned_filter['mashup_keyword'].notna()]['mashup_keyword'].unique()\n",
    "print(delimiter_found_values)\n",
    "print(mashup_keyword_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define strength categories\n",
    "strong_delimiters = ['x', '//']\n",
    "weak_delimiters = ['vs', 'vs.', 'versus', '+', '/']\n",
    "\n",
    "strong_keywords = ['mashup', 'mash-up', 'mash up', 'bootleg']\n",
    "weak_keywords = ['remix', 'mix', 'edit', 'rework', 'blend', 'white label']\n",
    "\n",
    "def determine_mashup_strength(row):\n",
    "    delimiter = row['delimiter_found']\n",
    "    keyword = row['mashup_keyword']\n",
    "    \n",
    "    # Determine delimiter strength\n",
    "    if pd.isna(delimiter) or delimiter is None:\n",
    "        delimiter_strength = None\n",
    "    elif delimiter in strong_delimiters:\n",
    "        delimiter_strength = 'strong'\n",
    "    elif delimiter in weak_delimiters:\n",
    "        delimiter_strength = 'weak'\n",
    "    else:\n",
    "        delimiter_strength = None  # Unknown delimiter\n",
    "    \n",
    "    # Determine keyword strength\n",
    "    if pd.isna(keyword) or keyword is None:\n",
    "        keyword_strength = None\n",
    "    elif keyword in strong_keywords:\n",
    "        keyword_strength = 'strong'\n",
    "    elif keyword in weak_keywords:\n",
    "        keyword_strength = 'weak'\n",
    "    else:\n",
    "        keyword_strength = None  # Unknown keyword\n",
    "    \n",
    "    # Combine strengths\n",
    "    if delimiter_strength == 'strong' and keyword_strength == 'strong':\n",
    "        return 'very strong'\n",
    "    elif delimiter_strength == 'strong' and keyword_strength == 'weak':\n",
    "        return 'strong'\n",
    "    elif delimiter_strength == 'weak' and keyword_strength == 'strong':\n",
    "        return 'strong'\n",
    "    elif delimiter_strength == 'weak' and keyword_strength == 'weak':\n",
    "        return 'weak'\n",
    "    elif delimiter_strength == 'strong' and keyword_strength is None:\n",
    "        return 'weak'\n",
    "    elif delimiter_strength == 'weak' and keyword_strength is None:\n",
    "        return 'very weak'\n",
    "    elif delimiter_strength is None and keyword_strength == 'strong':\n",
    "        return 'weak'\n",
    "    elif delimiter_strength is None and keyword_strength == 'weak':\n",
    "        return 'very weak'\n",
    "    else:\n",
    "        return 'very weak'  # Both None or any other case\n",
    "\n",
    "# Apply the function to create the mashup_strength column\n",
    "cleaned_filter['mashup_strength'] = cleaned_filter.apply(determine_mashup_strength, axis=1)\n",
    "\n",
    "# Show the distribution\n",
    "print(\"Mashup strength distribution:\")\n",
    "print(cleaned_filter['mashup_strength'].value_counts())\n",
    "\n",
    "# Show some examples\n",
    "print(\"\\nExamples by strength:\")\n",
    "for strength in ['very strong', 'strong', 'weak', 'very weak']:\n",
    "    print(f\"\\n{strength.upper()}:\")\n",
    "    examples = cleaned_filter[cleaned_filter['mashup_strength'] == strength][['song_name', 'delimiter_found', 'mashup_keyword', 'mashup_strength']].head(3)\n",
    "    if len(examples) > 0:\n",
    "        print(examples)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_filter.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert DataFrame to list of dictionaries (without index) and save to JSONL\n",
    "cleaned_filter_records = cleaned_filter.to_dict('records')\n",
    "\n",
    "print(f\"Total records to save: {len(cleaned_filter_records)}\")\n",
    "\n",
    "# Save to JSONL file\n",
    "write_jsonl(cleaned_filter_records, \"/home/sara/task_data/cleaned_mashup_data_wout_ws.jsonl\")\n",
    "\n",
    "print(\"Saved to /home/sara/task_data/cleaned_mashup_data.jsonl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24",
   "metadata": {},
   "outputs": [],
   "source": [
    "discogs_mashups = read_jsonl(\"mashup_discogs_match.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25",
   "metadata": {},
   "outputs": [],
   "source": [
    "discogs_mashups[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26",
   "metadata": {},
   "outputs": [],
   "source": [
    "title_to_data = {}\n",
    "for m in discogs_mashups:\n",
    "    title = m['original_entry']['title']\n",
    "    album = m['original_entry']['album']\n",
    "    artist = m['original_entry']['artist'][0]['name'] if 'artist' in m['original_entry'] else \" \"\n",
    "    title = title + \"---\" + album + \"---\" + str(artist)\n",
    "    if title not in title_to_data:\n",
    "        title_to_data[title] = []\n",
    "    title_to_data[title].append(m['original_entry'])\n",
    "\n",
    "len(title_to_data)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = read_jsonl(\"/home/sara/task_data/cleaned_mashup_data_wout_ws.jsonl\")\n",
    "x[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28",
   "metadata": {},
   "outputs": [],
   "source": [
    "discogs_final_data = []\n",
    "\n",
    "for m in x:\n",
    "    title = m['song_name']\n",
    "    data_source = m['data_source']\n",
    "    album = m['album_name'] if m['album_name'] is not None else \" \"\n",
    "    artist = m['artists'][0] if m['artists'] is not None else \" \"\n",
    "    title = title + \"---\" + album + \"---\" + str(artist)\n",
    "    if data_source == 'discogs' and title in title_to_data:\n",
    "        more_data = title_to_data[title][0]\n",
    "        m['s3_filepath'] = more_data['s3_filepath']\n",
    "    else:\n",
    "        m['s3_filepath'] = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30",
   "metadata": {},
   "outputs": [],
   "source": [
    "xx = read_jsonl(\"/home/sara/task_data/cleaned_mashup_data_wout_ws.jsonl\")\n",
    "print(len(xx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31",
   "metadata": {},
   "outputs": [],
   "source": [
    "write_jsonl(x, \"/home/sara/task_data/cleaned_mashup_data_wout_ws_discogs_s3.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_clean",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
