{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from suno_utils.utils.text import read_jsonl, write_jsonl\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import glob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all JSONL files from the traxsource_metadata directory\n",
    "traxsource_dir = \"/home/sara/traxsource_metadata/\"\n",
    "jsonl_files = sorted(glob.glob(f\"{traxsource_dir}/*.jsonl\"))\n",
    "print(f\"Found {len(jsonl_files)} JSONL files\")\n",
    "print(f\"First few files: {jsonl_files[:5]}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read and combine all JSONL files\n",
    "all_data = []\n",
    "\n",
    "for file_path in jsonl_files:\n",
    "    print(f\"Reading {Path(file_path).name}...\")\n",
    "    data = read_jsonl(file_path)\n",
    "    all_data.extend(data)\n",
    "    print(f\"  Added {len(data)} records\")\n",
    "\n",
    "print(f\"\\nTotal records: {len(all_data)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to DataFrame for easier analysis\n",
    "df = pd.DataFrame(all_data)\n",
    "print(f\"DataFrame shape: {df.shape}\")\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show column names and data types\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional: Save combined data to a single JSONL file\n",
    "write_jsonl(all_data, \"/home/sara/task_data/traxsource_metadata_raw.jsonl\")\n",
    "# print(\"Combined data saved to traxsource_metadata_combined.jsonl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_clean",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}