{ "cells": [ { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "import json\n", "import pandas as pd\n", "from suno_utils.utils.text import read_jsonl\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "from collections import defaultdict" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "labelbox_path = \"outputs/Export project - gender-prompt-adherence3 - 4_14_2025.ndjson\"\n", "labelbox_data = read_jsonl(labelbox_path)\n", "\n", "metadata_filepath = \"outputs/metadata-gender-adherence-20250404.json\"\n", "metadata = json.load(open(metadata_filepath, \"r\"))" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "results = []\n", "for row in labelbox_data:\n", " row_id = row[\"data_row\"][\"global_key\"]\n", " ground_truth = metadata[row_id][\"orig_pref\"].upper()\n", " genre = metadata[row_id][\"genre\"].lower()\n", " model = metadata[row_id][\"model\"]\n", " style_prompt = metadata[row_id][\"tags\"]\n", "\n", " projects = list(row[\"projects\"].keys())\n", " assert len(projects) == 1\n", " project_id = projects[0]\n", " project = row[\"projects\"][project_id]\n", " labels = project[\"labels\"]\n", " for label in labels: # represents each rating of the row, should be consensus count\n", " classifications = label[\"annotations\"][\"classifications\"]\n", " assert len(classifications) == 1 # should equal expected question count per data row\n", " for c in classifications:\n", " question = c[\"name\"]\n", " answer = c[\"radio_answer\"][\"value\"]\n", "\n", " results.append(\n", " {\n", " \"row_id\": row_id,\n", " \"ground_truth\": ground_truth.lower(),\n", " \"genre\": genre,\n", " \"style\": style_prompt,\n", " \"model\": model,\n", " \"question\": question,\n", " \"answer\": answer.lower(),\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
row_idground_truthgenrestylemodelquestionanswer
0e8c586f7-2759-48ec-b69f-dfc02e7378ce_gender-ad...malehip hopOld-School Hip-Hop, turntablism, heavy drums a...aukWhat gender is the singer?male
1874f545a-707e-4d52-8114-fd61f0cd167c_gender-ad...maleelectronicDubstep,rap, reggae, remix, heavy sub bassaukWhat gender is the singer?male
2c3357978-4a05-47d0-b483-1534099130cc_gender-ad...malerockhigh energy dynamic epic hardrock ballad deep ...v4What gender is the singer?male
375437846-c025-49a8-b4a5-c0283450d7c5_gender-ad...malepopsynth pop, alternative rock, indie rock, pop-p...v4What gender is the singer?male
40aa50197-b1b4-4073-a837-c10dce8f37b2_gender-ad...femalehip hopOld-School Hip-Hop, turntablism, heavy drums a...auk_augWhat gender is the singer?male
\n", "
" ], "text/plain": [ " row_id ground_truth genre \\\n", "0 e8c586f7-2759-48ec-b69f-dfc02e7378ce_gender-ad... male hip hop \n", "1 874f545a-707e-4d52-8114-fd61f0cd167c_gender-ad... male electronic \n", "2 c3357978-4a05-47d0-b483-1534099130cc_gender-ad... male rock \n", "3 75437846-c025-49a8-b4a5-c0283450d7c5_gender-ad... male pop \n", "4 0aa50197-b1b4-4073-a837-c10dce8f37b2_gender-ad... female hip hop \n", "\n", " style model \\\n", "0 Old-School Hip-Hop, turntablism, heavy drums a... auk \n", "1 Dubstep,rap, reggae, remix, heavy sub bass auk \n", "2 high energy dynamic epic hardrock ballad deep ... v4 \n", "3 synth pop, alternative rock, indie rock, pop-p... v4 \n", "4 Old-School Hip-Hop, turntablism, heavy drums a... auk_aug \n", "\n", " question answer \n", "0 What gender is the singer? male \n", "1 What gender is the singer? male \n", "2 What gender is the singer? male \n", "3 What gender is the singer? male \n", "4 What gender is the singer? male " ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.DataFrame(results)\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [], "source": [ "def score_model(df, model_name):\n", " model_rows = df[df.model == model_name]\n", " correct_responses = model_rows[model_rows.answer == model_rows.ground_truth]\n", " print(f\"Overall accuracy: {correct_responses.size / model_rows.size}\")\n", "\n", " answer_counts = model_rows[\"answer\"].value_counts()\n", " print(answer_counts)\n", "\n", "\n", "def score_model_by_gender(df, model_name):\n", " model_rows = df[df.model == model_name]\n", " male_rows = model_rows[model_rows.ground_truth == \"male\"]\n", " female_rows = model_rows[model_rows.ground_truth == \"female\"]\n", "\n", " correct_male = male_rows[male_rows.answer == male_rows.ground_truth]\n", " correct_female = female_rows[female_rows.answer == female_rows.ground_truth]\n", " print(f\"Male accuracy: {correct_male.size / male_rows.size}\")\n", " print(f\"Female accuracy: {correct_female.size / female_rows.size}\")" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overall accuracy: 0.8822222222222222\n", "answer\n", "male 236\n", "female 214\n", "Name: count, dtype: int64\n", "Male accuracy: 0.9066666666666666\n", "Female accuracy: 0.8577777777777778\n" ] } ], "source": [ "score_model(df, \"v4\")\n", "score_model_by_gender(df, \"v4\")" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overall accuracy: 0.6977777777777778\n", "answer\n", "male 186\n", "female 167\n", "no singer present 97\n", "Name: count, dtype: int64\n", "Male accuracy: 0.7022222222222222\n", "Female accuracy: 0.6933333333333334\n" ] } ], "source": [ "score_model(df, \"auk\")\n", "score_model_by_gender(df, \"auk\")" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overall accuracy: 0.8022222222222222\n", "answer\n", "male 216\n", "female 174\n", "no singer present 60\n", "Name: count, dtype: int64\n", "Male accuracy: 0.8444444444444444\n", "Female accuracy: 0.76\n" ] } ], "source": [ "score_model(df, \"auk_aug\")\n", "score_model_by_gender(df, \"auk_aug\")" ] } ], "metadata": { "kernelspec": { "display_name": "suno_clean", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 2 }