{ "cells": [ { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def analyze_forced_choice(count_a, count_b):\n", " from scipy import stats\n", " \n", " total = count_a + count_b\n", " prop_a = count_a / total\n", " prop_b = count_b / total\n", " \n", " # Binomial test against 50/50 null hypothesis\n", " p_value = stats.binomtest(count_b, total, p=0.5).pvalue\n", " \n", " return {\n", " 'proportion_a': prop_a,\n", " 'proportion_b': prop_b,\n", " 'difference': prop_b - prop_a,\n", " 'p_value': p_value\n", " }\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# load json\n", "#results_filepath = \"/home/christian/code/christian/metadata/go_listen/ab-test-20250127194502875433.json\"\n", "#results_filepath = \"/home/christian/code/christian/metadata/go_listen/ab-test-20250317123835591728.json\"\n", "results_filepath = \"/home/christian/code/christian/metadata/go_listen/ab-test-20250403165845956536.json\"\n", "with open(results_filepath, \"r\") as f:\n", " results = json.load(f)\n", "\n", "\n", "print(f\"Found {len(results)} participant(s)\")\n", "#model_a = \"dit_v2_dpo_t2_v1_3k\"\n", "#model_b = \"dit_v2_dpo_2025-01-26_02-13-54_s2704\"\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "a_master = 0 # we assume prod is a master\n", "b_master = 0 # we assume test (mut) is b master\n", "\n", "model_a = \"skip_1\"\n", "model_b = \"skip_4\"\n", "\n", "ingore_model = \"skip_2\"\n", "\n", "item_results = {}\n", "user_results = {}\n", "\n", "for user_idx, result in enumerate(results):\n", " items = result[\"items\"]\n", " name = items[0][\"questionControl\"][\"value\"]\n", " print(name)\n", " prod = {}\n", " for item_idx, item in enumerate(items[1:]):\n", " example = item[\"example\"]\n", " selection = example[\"fields\"][0][\"value\"]\n", " medias = example[\"medias\"]\n", " a = medias[0][\"filename\"]\n", " b = medias[1][\"filename\"]\n", "\n", " if ingore_model in a or ingore_model in b:\n", " continue\n", "\n", " # lets ensure that a and b contain either model_a or model_b\n", " if model_a not in a and model_a not in b:\n", " raise ValueError(f\"model_a not in a or b: {model_a} {a} {b}\")\n", " if model_b not in a and model_b not in b:\n", " raise ValueError(f\"model_b not in a or b: {model_b} {a} {b}\")\n", "\n", " if user_idx not in user_results:\n", " user_results[user_idx] = []\n", "\n", " if item_idx not in item_results:\n", " item_results[item_idx] = {model_a: [], model_b: []}\n", "\n", " if model_a in a:\n", " if selection == \"A\":\n", " a_master += 1\n", " user_results[user_idx].append(model_a)\n", " item_results[item_idx][model_a].append(name)\n", " #user_results[user_idx][\"A\"] += 1\n", " else:\n", " b_master += 1 \n", " user_results[user_idx].append(model_b)\n", " item_results[item_idx][model_b].append(name)\n", " #user_results[user_idx][\"B\"] += 1\n", " elif model_b in a:\n", " if selection == \"A\":\n", " b_master += 1\n", " user_results[user_idx].append(model_b)\n", " item_results[item_idx][model_b].append(name)\n", " #user_results[user_idx][\"A\"] += 1\n", " else:\n", " a_master += 1\n", " user_results[user_idx].append(model_a)\n", " item_results[item_idx][model_a].append(name)\n", " #user_results[user_idx][\"B\"] += 1\n", " else:\n", " print(f\"Unknown model: {a} or {b}\")\n", "\n", "\n", " # comments\n", " #comment = example[\"fields\"][1][\"value\"]\n", " #if comment is not None:\n", " # print(item_idx+1, user_results[user_idx][-1])\n", " # print(comment)\n", " \n", "\n", " #print(user_results[user_idx][-1])\n", " # count the number of times each model was chosen\n", " model_counts = {}\n", " for model in user_results[user_idx]:\n", " if model not in model_counts:\n", " model_counts[model] = 0\n", " model_counts[model] += 1\n", " for model, count in model_counts.items():\n", " print(f\"{model}: {count} ({count/len(user_results[user_idx])*100:.2f}%)\")\n", " print()\n", " \n", " \n", "\n", "print(f\"{model_a}: {a_master} {model_b}: {b_master} Total: {a_master + b_master}\")\n", "a_win_rate = a_master / (a_master + b_master)\n", "b_win_rate = b_master / (a_master + b_master)\n", "#print(f\"Prod win rate: {prod_win_rate*100:.2f}%\")\n", "print(f\"{model_a} win rate: {a_win_rate*100:.2f}% ({model_b} win rate: {b_win_rate*100:.2f}%) ({(a_win_rate - b_win_rate) *100:.2f}%)\")\n", "\n", "# now go through and look for agreement on each item\n", "for item_idx, item in item_results.items():\n", " print(item_idx+1, item)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for user_idx, user in user_results.items():\n", " user_pref = \"new\" if user[\"A\"] > user[\"B\"] else \"prod\"\n", " print(f\"User {user_idx+1:2d}: pref = {user_pref:4s} | new: {user['A']:2d} prod: {user['B']:2d}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "print(analyze_forced_choice(a_master, b_master))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "51\n", "49" ] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }