{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from suno_utils.utils.text import read_jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labelbox_path = \"Export  project - v4_prod-vs-v35_prod-basic-preference - 2_17_2025.ndjson\"\n",
    "labelbox_data = read_jsonl(labelbox_path)\n",
    "\n",
    "metadata_filepath = \"outputs/metadata-v4_prod-vs-v35_prod-20250214.json\"\n",
    "metadata = json.load(open(metadata_filepath, \"r\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "for row in labelbox_data:\n",
    "    row_id = row[\"data_row\"][\"global_key\"]\n",
    "    ground_truth = metadata[row_id][\"orig_pref\"].upper()\n",
    "\n",
    "    projects = list(row[\"projects\"].keys())\n",
    "    assert len(projects) == 1\n",
    "    project_id = projects[0]\n",
    "    project = row[\"projects\"][project_id]\n",
    "    labels = project[\"labels\"]\n",
    "    for label in labels:  # represents each rating of the row, should be consensus count\n",
    "        classifications = label[\"annotations\"][\"classifications\"]\n",
    "        assert len(classifications) == 1\n",
    "        for c in classifications:\n",
    "            question = c[\"name\"]\n",
    "            answer = c[\"radio_answer\"][\"value\"]\n",
    "\n",
    "            # if ground_truth == \"B\":\n",
    "            #    if answer == \"A\":\n",
    "            #        answer = \"B\"\n",
    "            #    elif answer == \"B\":\n",
    "            #        answer = \"A\"\n",
    "\n",
    "            results.append(\n",
    "                {\"row_id\": row_id, \"ground_truth\": ground_truth, \"question\": question, \"answer\": answer}\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(results)\n",
    "print(df.count())\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall Preference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preference_counts = df[\"answer\"].value_counts()\n",
    "preference_percentages = df[\"answer\"].value_counts(normalize=True) * 100\n",
    "\n",
    "print(\"Overall Preference\")\n",
    "print(f\"v4: {preference_counts['A']} votes, {preference_percentages['A']}%\")\n",
    "print(f\"v3.5: {preference_counts['B']} votes, {preference_percentages['B']}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Consistency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First, find which task_ids have consistent preferences\n",
    "task_consistency = df.groupby(\"row_id\")[\"answer\"].nunique()\n",
    "consistent_tasks = task_consistency[task_consistency == 1].index\n",
    "\n",
    "# Filter the dataframe to only include those task_ids\n",
    "consistent_df = df[df[\"row_id\"].isin(consistent_tasks)]\n",
    "\n",
    "# Now calculate the preference distribution\n",
    "consistent_preferences = consistent_df[\"answer\"].value_counts()\n",
    "consistent_percentages = consistent_df[\"answer\"].value_counts(normalize=True) * 100\n",
    "\n",
    "print(\"Preference on Tasks with Perfect Consistency\")\n",
    "print(f\"v4: {consistent_preferences['A']} votes, {consistent_percentages['A']}%\")\n",
    "print(f\"v3.5: {consistent_preferences['B']} votes, {consistent_percentages['B']}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inconsistent_tasks = task_consistency[task_consistency != 1].index\n",
    "inconsistent_df = df[df[\"row_id\"].isin(inconsistent_tasks)]\n",
    "inconsistent_preferences = inconsistent_df[\"answer\"].value_counts()\n",
    "inconsistent_percentages = inconsistent_df[\"answer\"].value_counts(normalize=True) * 100\n",
    "\n",
    "print(\"\")\n",
    "print(\"Preference on Tasks with Imperfect Consistency\")\n",
    "print(f\"v4: {inconsistent_preferences['A']} votes, {inconsistent_percentages['A']}%\")\n",
    "print(f\"v3.5: {inconsistent_preferences['B']} votes, {inconsistent_percentages['B']}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_tasks = df[\"row_id\"].nunique()\n",
    "consistent_task_count = len(consistent_tasks)\n",
    "inconsistent_task_count = total_tasks - consistent_task_count\n",
    "\n",
    "print(f\"Total tasks: {total_tasks}\")\n",
    "print(f\"Tasks with agreement: {consistent_task_count}\")\n",
    "print(f\"Tasks with disagreement: {inconsistent_task_count}\")\n",
    "print(f\"{consistent_task_count / total_tasks * 100}% task consistency\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_clean",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}