{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "5fa9236d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "e91fbb75", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "f6938960", "metadata": {}, "outputs": [], "source": [ "# LICENSED_PODCAST_INFO = [\n", "# (189609, \"BOOTHCAST\", None),\n", "# (257070, \"Let's Talk About Mental Health\", None),\n", "# (2082586, \"Ignite Purpose\", None),\n", "# (965640, \"Tokyo Munchies\", None),\n", "# (970010, \"The Truth About Coaching\", None),\n", "# (\n", "# 2573974, \n", "# \"AgriAdventures\", \n", "# set([\n", "# \"58ca5e3c-2a46-4693-acd4-cb6c6099f339\",\n", "# \"10c24ac5-1ce9-4b37-8918-4e24b5375b6a\",\n", "# \"f97f9b82-ee49-4aae-a375-61ce7d97409b\",\n", "# ])\n", "# ),\n", "# (1181554, \"The Blacklist Sessions\", None),\n", "# (802920, \"Soundtrack\", None),\n", "# (4952455, \"The Empowered Women's Podcast\", None),\n", "# (535850, \"The Effortless Swimming Podcast\", None),\n", "# (739552, \"World Journeys\", None),\n", "# (\n", "# 162254, \n", "# \"Coexisting With Nonhuman Animals\",\n", "# set([\n", "# \"tag:blogger.com,1999:blog-756349456522903168.post-4728220547061739117\",\n", "# ])\n", "# ),\n", "# (966750, \"Legends with Bevo\", None),\n", "# ]\n", "\n", "# # fire\n", "# # dropkicks\n", "# # online radio australia" ] }, { "cell_type": "code", "execution_count": null, "id": "2dae7944", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "274ab146", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2fbe4d31", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2bc355d8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "9887d690", "metadata": {}, "source": [ "## Assemble the audio data" ] }, { "cell_type": "code", "execution_count": null, "id": "4f5d3f5c", "metadata": {}, "outputs": [], "source": [ "python /home/georg/code/glockenspiel/suno_utils/suno_utils/scripts/fetch_podcasts.py \\\n", " --podcast_db_filepath=\"/mnt/data-ssd-1/data/podcasts/meta/podcastindex_feeds.db\" \\\n", " --podcast_ids=\"189609,257070,2082586,965640,970010,2573974,1181554,802920,4952455,535850,739552, 966750\" \\\n", " --output_dir=\"/mnt/data-ssd-1/data/podcasts/audio\"\n", "\n", "# --episode_ids=\"tag:blogger.com,1999:blog-756349456522903168.post-4728220547061739117\"\n", "# --last_n_episodes=-1" ] }, { "cell_type": "code", "execution_count": null, "id": "12657788", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "46e410ac", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "251055f6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "8f40160c", "metadata": {}, "source": [ "### (optional) analyze" ] }, { "cell_type": "code", "execution_count": 4, "id": "e3ed8b1d", "metadata": {}, "outputs": [], "source": [ "# https://github.com/NVIDIA/NeMo/blob/main/tutorials/speaker_tasks/Speaker_Diarization_Inference.ipynb" ] }, { "cell_type": "code", "execution_count": 12, "id": "1253b349", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAACtCAYAAAAKyYJgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPCElEQVR4nO3de7BuZV0H8O+Pi9lgIgoRAXUsCcMcEc8YRDmGCiqOimHiWGJjY82YaTWlNs5wMC2wEikvM42XTmp4QR0dLJBBvFQiHi7KTQQLBk5cVBRT84L8+uNd57g93M7J/e732ezPZ+add61nPXvt593rd961z3ev9bzV3QEAAABgLDstegAAAAAA3JHQBgAAAGBAQhsAAACAAQltAAAAAAYktAEAAAAYkNAGAAAAYEBCGwAAAIABCW0AAAAABiS0AQAAABiQ0AYAAABgQEIbAAAAgAEJbSZV9byqev0y7/OBVXV2VV01Pe+xnPu/t5rTsXhmVV1WVbdX1frl3DcAAADMg9BmGVTVznex6WVJzunuA5KcM60zR3dzLC5N8owkn1jB4QAAAMD/26oKbapqt6r6cFV9tqourapnVdU1VfWaqrqkqs6vqodMffeqqvdV1Wemx+FT+6Or6lNVdVFV/UdVHXgn3+foqc+eVXXktHxhVb23qu439bmmqk6uqguTPPMuhvy0JBun5Y1Jnr7cP5NFWW3Horuv6O4r5/gjAQAAgGW1y4/yxYeecNaGJCcsz1CSJCeed+JRG+5m+xOT/Hd3H50kVbV7kpOT3NrdD6+q5yZ5XZKnJDk1ySnd/W9V9TNJzkryi0k+n+TXuvu2qnp8kr9M8htbvkFVHZPkj5M8OcnOSV6R5PHd/c2qeum07ZVT96909yF3M969u/uGafnGJHtv589hh2zed/8NWebjsO/m6zbcQ5/VdiwAAABgVfmRQpsFuCTJ31bVyUnO6O5PVlWSnDZtPy3JKdPy45McNG1PkvtPV2bsnmRjVR2QpJPsumT/RyRZn+TI7v56VT0lyUFJ/n3az32SfGpJ/3dv78C7u6uqt/uVjm/VHgsAAABYDVZVaNPdX6iqQzK78uJVVXXOlk1Lu03POyU5tLu/vXQf0wS353b3MVW1LsnHlmz+YpKfS/ILSTYlqSRnd/ez72JI37yHId9UVft09w1VtU+Sm++h/6qxCo8FAAAArCo/Umgz3cq0YVlGsh2q6qeT3NLd76iqryX53WnTs5KcND1vufriI0lelOSvp689uLsvzuzqjs1Tn+dt8y2uTfKnSd5fVc9Mcl6SN1TVQ7r76qraLcm+3f2F7Rzyh5IcP43t+CQf3P5Xu/2mW5k2zGPfd2UVHgsAAABYVVbVRMRJHp7k/Kq6OLM5XF41te9RVZ9L8uIkfzS1/WGS9VX1uaq6PMnvT+2vSfJXVXVR7iS06u7PJ3lOkvcmuX9mYcJp0/4/leShOzDek5I8oaquyuwWoZN24GtHt6qORVUdU1XXJzksyYer6qwdeK0AAACw4qp7dU+zUlXXJFnf3V9e9FjWOscCAAAAls9qu9IGAAAAYE1Y9VfajKCq3pDk8G2aT+3uty1iPGuZYwEAAMC9hdAGAAAAYEBujwIAAAAYkNAGAAAAYEB3+Jjlu7Pnnnv2unXr5jQUAAAAgLXnggsu+HJ377Vt+w6FNuvWrcumTZuWb1QAAAAAa1xVXXtn7W6PAgAAABiQ0AYAAABgQEIbAAAAgAEJbQAAAAAGJLQBAAAAGJDQBgAAAGBAQhsAAACAAQltAAAAAAYktAEAAAAYkNAGAAAAYEBCGwAAAIABCW0AAAAABiS0AQAAABiQ0AYAAABgQEIbAAAAgAEJbQAAAAAGJLQBAAAAGJDQBgAAAGBAQhsAAACAAQltAAAAAAYktAEAAAAYkNAGAAAAYEBCGwAAAIABCW0AAAAABiS0AQAAABiQ0AYAAABgQEIbAAAAgAEJbQAAAAAGJLQBAAAAGJDQBgAAAGBAQhsAAACAAQltAAAAAAYktAEAAAAYkNAGAAAAYEBCGwAAAIABCW0AAAAABiS0AQAAABiQ0AYAAABgQEIbAAAAgAEJbQAAAAAGJLQBAAAAGJDQBgAAAGBAQhsAAACAAQltAAAAAAYktAEAAAAYkNAGAAAAYEBCGwAAAIAB7VBo8/2bbvqh9de/+u1Jko/9wSu2rj/9tR9fpqEBAAAArF07FNrcvk1o847v/mSS5IAPbNy6fuOt316moQEAAACsXW6PAgAAABiQ0AYAAABgQDsc2mzed/+tjyQ59ISztrYDAAAAsDxcaQMAAAAwIKENAAAAwIB2OLTZd/N1Wx9Jct6JR21tBwAAAGB5uNIGAAAAYEBCGwAAAIABCW0AAAAABrRDoc1Oe+/9Q+u/dZ+bkyRXHXP81vWf2v2+yzQ0AAAAgLWrunu7O69fv743bdo0x+EAAAAArC1VdUF3r9+23e1RAAAAAAMS2gAAAAAMSGgDAAAAMCChDQAAAMCAhDYAAAAAAxLaAAAAAAxIaAMAAAAwIKENAAAAwICENgAAAAADEtoAAAAADEhoAwAAADAgoQ0AAADAgIQ2AAAAAAMS2gAAAAAMSGgDAAAAMCChDQAAAMCAhDYAAAAAAxLaAAAAAAxIaAMAAAAwIKENAAAAwICENgAAAAADEtoAAAAADEhoAwAAADAgoQ0AAADAgIQ2AAAAAAMS2gAAAAAMSGgDAAAAMCChDQAAAMCAhDYAAAAAAxLaAAAAAAxIaAMAAAAwIKENAAAAwICENgAAAAADEtoAAAAADEhoAwAAADAgoQ0AAADAgIQ2AAAAAAMS2gAAAAAMSGgDAAAAMCChDQAAAMCAhDYAAAAAAxLaAAAAAAxIaAMAAAAwIKENAAAAwICENgAAAAADEtoAAAAADEhoAwAAADCg6u7t71z1pSTXzm84cAd7JvnyogfBmqHeWGlqjpWm5lhpao6VpuZYactVcz/b3Xtt27hDoQ2stKra1N3rFz0O1gb1xkpTc6w0NcdKU3OsNDXHSpt3zbk9CgAAAGBAQhsAAACAAQltGN0/LHoArCnqjZWm5lhpao6VpuZYaWqOlTbXmjOnDQAAAMCAXGkDAAAAMCChDSuqqt5aVTdX1aVL2h5YVWdX1VXT8x5Te1XV31XV1VX1uao6ZMnXHD/1v6qqjl/Ea2F1qKr9q+rcqrq8qi6rqhdP7eqOuaiq+1bV+VX12anmTpzaH1xVn55q691VdZ+p/cem9aun7euW7OvlU/uVVXXUgl4Sq0BV7VxVF1XVGdO6emOuquqaqrqkqi6uqk1Tm3Mrc1FVD6iq06vq81V1RVUdpt6Yl6o6cHpv2/L4elW9ZFE1J7Rhpf1jkidu0/ayJOd09wFJzpnWk+RJSQ6YHi9I8qZk9gtBkhOS/HKSRyc5Ycs/GLgTtyX5k+4+KMmhSV5YVQdF3TE/30lyRHc/IsnBSZ5YVYcmOTnJKd39kCRfTfL8qf/zk3x1aj9l6pepTo9L8rDM3jffWFU7r+QLYVV5cZIrlqyrN1bCr3f3wUs+6ta5lXk5NcmZ3f3QJI/I7P1OvTEX3X3l9N52cJJHJflWkg9kQTUntGFFdfcnktyyTfPTkmycljcmefqS9n/qmfOSPKCq9klyVJKzu/uW7v5qkrNzxyAIkiTdfUN3Xzgt/09mJ/l9o+6Yk6l2vjGt7jo9OskRSU6f2retuS21eHqSx1VVTe3v6u7vdPd/Jbk6sxM+/JCq2i/J0UnePK1X1BuL4dzKsquq3ZM8JslbkqS7v9vdX4t6Y2U8LskXu/vaLKjmhDaMYO/uvmFavjHJ3tPyvkmuW9Lv+qntrtrhbk23ATwyyaej7pij6VaVi5PcnNkJ+otJvtbdt01dltbP1tqatt+a5EFRc2y/1yX5syS3T+sPinpj/jrJR6rqgqp6wdTm3Mo8PDjJl5K8bboN9M1VtVvUGyvjuCSnTcsLqTmhDUPp2ceZ+Ugzll1V3S/J+5K8pLu/vnSbumO5dff3p0tq98vsaoWHLnZE3FtV1VOS3NzdFyx6LKw5v9rdh2R2W8ALq+oxSzc6t7KMdklySJI3dfcjk3wzP7gtJYl6Yz6m+eCemuS9225byZoT2jCCm6bLxzI93zy1b06y/5J++01td9UOd6qqds0ssHlnd79/alZ3zN10+fa5SQ7L7FLZXaZNS+tna21N23dP8pWoObbP4UmeWlXXJHlXZrdFnRr1xpx19+bp+ebM5np4dJxbmY/rk1zf3Z+e1k/PLMRRb8zbk5Jc2N03TesLqTmhDSP4UJItM2kfn+SDS9qfO83GfWiSW6fL0c5KcmRV7TFN5HTk1AZ3MM3V8JYkV3T3a5dsUnfMRVXtVVUPmJZ/PMkTMptL6dwkx07dtq25LbV4bJKPTn+9+VCS42r2aT8Pzmxyu/NX5EWwanT3y7t7v+5el9kl3B/t7udEvTFHVbVbVf3EluXMzomXxrmVOejuG5NcV1UHTk2PS3J51Bvz9+z84NaoZEE1t8s9d4HlU1WnJXlskj2r6vrMZtM+Kcl7qur5Sa5N8ptT939J8uTMJkP8VpLfSZLuvqWq/iLJZ6Z+r+zubSc3hi0OT/LbSS6Z5hhJkj+PumN+9kmycfrknZ2SvKe7z6iqy5O8q6peleSiTBMqTs9vr6qrM5uo/bgk6e7Lquo9mf1ieluSF3b391f4tbB6vTTqjfnZO8kHZn8XyS5J/rm7z6yqz8S5lfl4UZJ3Trer/GdmNbRT1BtzMgXST0jye0uaF/L/h5r9cQUAAACAkbg9CgAAAGBAQhsAAACAAQltAAAAAAYktAEAAAAYkNAGAAAAYEBCGwBgeFX1oKq6eHrcWFWbp+VvVNUbFz0+AIB58JHfAMCqUlUbknyju/9m0WMBAJgnV9oAAKtWVT22qs6YljdU1caq+mRVXVtVz6iq11TVJVV1ZlXtOvV7VFV9vKouqKqzqmqfxb4KAIA7J7QBAO5Nfj7JEUmemuQdSc7t7ocn+d8kR0/Bzd8nOba7H5XkrUlevajBAgDcnV0WPQAAgGX0r939vaq6JMnOSc6c2i9Jsi7JgUl+KcnZVZWpzw0LGCcAwD0S2gAA9ybfSZLuvr2qvtc/mLzv9sx+76kkl3X3YYsaIADA9nJ7FACwllyZZK+qOixJqmrXqnrYgscEAHCnhDYAwJrR3d9NcmySk6vqs0kuTvIrCx0UAMBd8JHfAAAAAANypQ0AAADAgIQ2AAAAAAMS2gAAAAAMSGgDAAAAMCChDQAAAMCAhDYAAAAAAxLaAAAAAAxIaAMAAAAwoP8D2sNL8pb1GzEAAAAASUVORK5CYII=\n", "text/plain": [ "" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from suno_utils.audio import Audio\n", "from suno_utils.tasks.diarization import diarize, plot_segments\n", "\n", "audio = Audio.from_file(\"/mnt/data-ssd-1/data/podcasts/audio/189609/cd9affaf-22ec-45b1-8da8-07e613d11877.m4a\")\n", "speaker_segments = diarize(audio.get_slice(to_s=60))\n", "convert_to_pyannote(speaker_segments)" ] }, { "cell_type": "code", "execution_count": 7, "id": "b19d3d29", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "0it [00:00, ?it/s]\n", "100%|██████████| 5/5 [02:40<00:00, 32.03s/it]\n", "100%|██████████| 5/5 [00:37<00:00, 7.59s/it]\n", "100%|██████████| 5/5 [01:10<00:00, 14.07s/it]\n", "100%|██████████| 5/5 [02:03<00:00, 24.67s/it]\n", "100%|██████████| 5/5 [02:29<00:00, 29.94s/it]\n", "100%|██████████| 5/5 [01:59<00:00, 23.86s/it]\n", "100%|██████████| 5/5 [02:24<00:00, 29.00s/it]\n", "100%|██████████| 5/5 [04:34<00:00, 54.84s/it]\n", "100%|██████████| 5/5 [02:50<00:00, 34.17s/it]\n", "100%|██████████| 5/5 [02:11<00:00, 26.36s/it]\n" ] } ], "source": [ "import pickle\n", "from suno_utils.audio.conversion import get_audio_properties\n", "import tqdm\n", "\n", "def _get_id_from_filepath(filepath):\n", " return \".\".join(filepath.split(\"/\")[-1].split(\".\")[:-1]).replace(\"__\", \"/\")\n", "\n", "diarization_data = {}\n", "tot_duration_h = 0 \n", "for dirpath, dirs, files in os.walk(\"/mnt/data-ssd-1/data/podcasts/audio/\"): \n", " for filename in tqdm.tqdm(files[:5]):\n", " filepath = os.path.join(dirpath, filename)\n", " try:\n", " _ = get_audio_properties(filepath)[\"duration_s\"]\n", " audio = Audio.from_file(filepath)\n", " speaker_segments = diarize(audio)\n", " diarization_data[_get_id_from_filepath(filepath)] = speaker_segments\n", " with open(\"tmp/diarization_data.pkl\", \"wb\") as f:\n", " pickle.dump(diarization_data, f)\n", " except:\n", " pass" ] }, { "cell_type": "code", "execution_count": null, "id": "e7dd2f05", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1f5ac0bc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d0178305", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "99e987f5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "54129db2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c00712bb", "metadata": {}, "source": [ "## Do rev transcription" ] }, { "cell_type": "code", "execution_count": null, "id": "f61c4a3d", "metadata": {}, "outputs": [], "source": [ "# select what we want and put transcripts into /mnt/data-ssd-1/data/podcasts/transcripts\n", "# and then manifest into gdata" ] }, { "cell_type": "code", "execution_count": null, "id": "4efba28a", "metadata": {}, "outputs": [], "source": [ "# how many episodes do we need per show\n", "# max 1h for each episode transcribed\n", "# randomly sample episodes" ] }, { "cell_type": "code", "execution_count": null, "id": "2b6a60f4", "metadata": {}, "outputs": [], "source": [ "# use all of agri\n", "# less:\n", "# Tokyo Munchies\n", "# The Truth About Coaching\n", "# Let's Talk About Mental Health\n", "# more:\n", "# Legends with Bevo\n", "# BOOTHCast\n", "# the Empowered Women's\n", "# effortless swimming" ] }, { "cell_type": "code", "execution_count": 4, "id": "7f9a456f", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "from suno_utils.utils.podcasts import get_episodes_to_label\n", "\n", "PODCAST_FULL_INFO = [\n", " (\n", " 2573974, \n", " \"AgriAdventures\", \n", " set([\n", " \"10c24ac5-1ce9-4b37-8918-4e24b5375b6a\",\n", " \"58ca5e3c-2a46-4693-acd4-cb6c6099f339\",\n", " \"f97f9b82-ee49-4aae-a375-61ce7d97409b\",\n", " ]),\n", " ), (\n", " 162254, \n", " \"Coexisting With Nonhuman Animals\", \n", " set([\n", " \"tag:blogger.com,1999:blog-756349456522903168.post-4728220547061739117\",\n", " ]),\n", " )\n", "]\n", "\n", "# id, name, min_hours\n", "PODCASTS_FRAGMENT_INFO = [\n", " (189609, \"BOOTHCAST\", 8),\n", " (257070, \"Let's Talk About Mental Health\", 4),\n", " (2082586, \"Ignite Purpose\", 4),\n", " (965640, \"Tokyo Munchies\", 4),\n", " (970010, \"The Truth About Coaching\", 4),\n", " (1181554, \"The Blacklist Sessions\", 4),\n", " (802920, \"Soundtrack\", 4),\n", " (4952455, \"The Empowered Women's Podcast\", 8),\n", " (535850, \"The Effortless Swimming Podcast\", 8),\n", " (739552, \"World Journeys\", 4),\n", " (966750, \"Legends with Bevo\", 8),\n", "]\n", "\n", "PODCAST_AUDIO_DIR = \"/mnt/data-ssd-1/data/podcasts/audio/\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "42b5796c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 2/2 [00:01<00:00, 1.69it/s]\n", "100%|██████████| 11/11 [00:11<00:00, 1.06s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "65.1 hours of content to label\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "rev_manifest = []\n", "\n", "for podcast_id, _, episode_id_subset in tqdm.tqdm(PODCAST_FULL_INFO):\n", " audio_dir = PODCAST_AUDIO_DIR + f\"{podcast_id}/\"\n", " annotation_manifest = get_episodes_to_label(\n", " audio_dir,\n", " min_episode_duration=5*60,\n", " min_total_duration_s=np.inf,\n", " max_per_episode_duration_s=np.inf,\n", " episode_id_subset=episode_id_subset,\n", " )\n", " rev_manifest.extend(annotation_manifest)\n", "\n", "for podcast_id, _, min_duration_h in tqdm.tqdm(PODCASTS_FRAGMENT_INFO):\n", " audio_dir = PODCAST_AUDIO_DIR + f\"{podcast_id}/\"\n", " annotation_manifest = get_episodes_to_label(\n", " audio_dir, \n", " shuffle=True,\n", " min_episode_duration=5*60,\n", " min_total_duration_s=min_duration_h*60*60,\n", " max_per_episode_duration_s=60*60,\n", " )\n", " rev_manifest.extend(annotation_manifest)\n", "\n", "tot_duration_s = np.sum([e[2] - e[1] for e in rev_manifest])\n", "print(f\"{round(tot_duration_s / 60 / 60, 1)} hours of content to label\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "9fb506e4", "metadata": {}, "outputs": [], "source": [ "import uuid\n", "from suno_utils.utils.text import write_jsonl\n", "\n", "manifest_data = [\n", " {\"filepath\": e[0], \"start_s\": e[1], \"end_s\": e[2], \"hotwords\": e[3], \"uuid\": str(uuid.uuid4())} \n", " for e in rev_manifest\n", "]\n", "write_jsonl(manifest_data, \"gdata_mate_rev_manifest.jsonl\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "a5db051d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\"filepath\": \"/mnt/data-ssd-1/data/podcasts/audio/2573974/58ca5e3c-2a46-4693-acd4-cb6c6099f339.m4a\", \"start_s\": 0, \"end_s\": 992.026, \"hotwords\": [\"Chris Williams Interview - Ambersun\", \"AgriAdventures\", \"Chris Williams\", \"Mount Compass\", \"the Victor Harbor Road\", \"Adelaide\", \"Chris\", \"Naracoorte\", \"Alpacas\", \"Adrienne\", \"the Royal Adelaide Show\", \"Ambersun Alpacas\", \"South America\", \"Finland\", \"Norway\", \"Sweeden\", \"Danemark\", \"China\", \"Indonesia\", \"Facebook\", \"AgriAdventures AgriAdventures\"], \"uuid\": \"fbf9d6c5-a122-42d4-84cc-15f43dcb72e3\"}\r\n", "{\"filepath\": \"/mnt/data-ssd-1/data/podcasts/audio/2573974/10c24ac5-1ce9-4b37-8918-4e24b5375b6a.m4a\", \"start_s\": 0, \"end_s\": 2702.27, \"hotwords\": [\"Daniel Motlop - Something\", \"Wild - Adelaide\", \"Daniel Tahlia Motlop\", \"Adelaide Central Market Australia\", \"the Mission and Vision of Daniel\", \"AgriAdventures\", \"the Adelaide Central Market Simone Berliat\", \"AgriAdventures AgriAdventures\"], \"uuid\": \"fa1d62a2-9a67-4def-a2e0-9c3d888cae69\"}\r\n" ] } ], "source": [ "!head -2 gdata_mate_rev_manifest.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "fefe4a92", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bdfddeb1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "881a2ee6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b14d5ae8", "metadata": {}, "source": [ "### save hours of podcasts for attribution" ] }, { "cell_type": "code", "execution_count": 14, "id": "b7c3f4dd", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import re\n", "from collections import Counter\n", "from suno_utils.utils.text import load_jsonl\n", "from suno_utils.audio.conversion import get_audio_properties\n", "\n", "\n", "PODCAST_FULL_INFO = [\n", " (\n", " 2573974, \n", " \"AgriAdventures\", \n", " set([\n", " \"10c24ac5-1ce9-4b37-8918-4e24b5375b6a\",\n", " \"58ca5e3c-2a46-4693-acd4-cb6c6099f339\",\n", " \"f97f9b82-ee49-4aae-a375-61ce7d97409b\",\n", " ]),\n", " ), (\n", " 162254, \n", " \"Coexisting With Nonhuman Animals\", \n", " set([\n", " \"tag:blogger.com,1999:blog-756349456522903168.post-4728220547061739117\",\n", " ]),\n", " )\n", "]\n", "\n", "# id, name, min_hours\n", "PODCASTS_FRAGMENT_INFO = [\n", " (189609, \"BOOTHCAST\", 8),\n", " (257070, \"Let's Talk About Mental Health\", 4),\n", " (2082586, \"Ignite Purpose\", 4),\n", " (965640, \"Tokyo Munchies\", 4),\n", " (970010, \"The Truth About Coaching\", 4),\n", " (1181554, \"The Blacklist Sessions\", 4),\n", " (802920, \"Soundtrack\", 4),\n", " (4952455, \"The Empowered Women's Podcast\", 8),\n", " (535850, \"The Effortless Swimming Podcast\", 8),\n", " (739552, \"World Journeys\", 4),\n", " (966750, \"Legends with Bevo\", 8),\n", "]\n", "\n", "podcast_info_map = {}\n", "for podcast_id, podcast_name, episodes in PODCAST_FULL_INFO:\n", " podcast_info_map[podcast_id] = {\"name\": podcast_name, \"episodes\": episodes}\n", "for podcast_id, podcast_name, episodes in PODCASTS_FRAGMENT_INFO:\n", " podcast_info_map[podcast_id] = {\"name\": podcast_name, \"episodes\": episodes}\n", "\n", "manifest_data = load_jsonl(\"gdata_mate_rev_manifest.jsonl\")" ] }, { "cell_type": "code", "execution_count": 52, "id": "b02c3675", "metadata": {}, "outputs": [], "source": [ "tot_duration_s = 0\n", "detail_attribution_data = []\n", "for m in manifest_data:\n", " podcast_id, episode_id = re.sub(r\"\\.[^\\.]+$\", \"\", m[\"filepath\"]).split(\"/\")[-2:]\n", " podcast_id = int(podcast_id)\n", " duration_s = m[\"end_s\"] - m[\"start_s\"]\n", " tot_duration_s += duration_s\n", " compat_fp = m[\"filepath\"].replace(\"/audio/\", \"/bk_audio/\")\n", " audio_info = get_audio_properties(compat_fp, attempt_using_header=False)\n", " true_duration_s = audio_info[\"duration_s\"]\n", " is_whole_episode = bool(np.abs(true_duration_s - duration_s) <= 1)\n", " podacast_info = podcast_info_map[podcast_id]\n", " detail_attribution_data.append({\n", " \"podcast_id\": podcast_id,\n", " \"podcast_name\": podacast_info[\"name\"],\n", " \"episode_id\": episode_id,\n", " \"duration_s\": round(duration_s, 1),\n", " \"transcribed_whole_episode\": is_whole_episode,\n", " })\n", " \n", "attribution_pct = Counter()\n", "for e in detail_attribution_data:\n", " attribution_pct[e[\"podcast_id\"]] += e[\"duration_s\"]\n", "attribution_pct = {k: round(v / tot_duration_s * 100, 2) for k, v in attribution_pct.items()}\n", "\n", "attribution_data = []\n", "for k, v in attribution_pct.items():\n", " podacast_info = podcast_info_map[k]\n", " attribution_data.append({\n", " \"podcast_id\": k,\n", " \"podcast_name\": podacast_info[\"name\"],\n", " \"used_pct\": v,\n", " })\n", "assert(int(round(np.sum([e[\"used_pct\"] for e in attribution_data]))) == 100)" ] }, { "cell_type": "code", "execution_count": 53, "id": "25266c28", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'podcast_id': 2573974, 'podcast_name': 'AgriAdventures', 'used_pct': 1.99},\n", " {'podcast_id': 162254,\n", " 'podcast_name': 'Coexisting With Nonhuman Animals',\n", " 'used_pct': 1.21}]" ] }, "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "source": [ "attribution_data[:2]" ] }, { "cell_type": "code", "execution_count": 54, "id": "ac3dc755", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "118\n" ] }, { "data": { "text/plain": [ "[{'podcast_id': 2573974,\n", " 'podcast_name': 'AgriAdventures',\n", " 'episode_id': '58ca5e3c-2a46-4693-acd4-cb6c6099f339',\n", " 'duration_s': 992.0,\n", " 'transcribed_whole_episode': True},\n", " {'podcast_id': 2573974,\n", " 'podcast_name': 'AgriAdventures',\n", " 'episode_id': '10c24ac5-1ce9-4b37-8918-4e24b5375b6a',\n", " 'duration_s': 2702.3,\n", " 'transcribed_whole_episode': True}]" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print(len(detail_attribution_data))\n", "detail_attribution_data[:2]" ] }, { "cell_type": "code", "execution_count": 58, "id": "90a5cf5b", "metadata": {}, "outputs": [], "source": [ "pd.DataFrame(attribution_data).to_csv(\"podcast_attribution.csv\", index=False)\n", "pd.DataFrame(detail_attribution_data).to_csv(\"episode_attribution.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "4fb8a92b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "18cfe42d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "79e9d36f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "66a9e150", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f0dbc4c4", "metadata": {}, "source": [ "### listen to manifest items" ] }, { "cell_type": "code", "execution_count": 39, "id": "1b358595", "metadata": {}, "outputs": [], "source": [ "import random\n", "from suno_utils.utils.text import load_jsonl\n", "from suno_utils.utils.notebook import Audio\n", "\n", "manifest_data = load_jsonl(\"/home/georg/notebooks/datasets/gdata_mate/gdata_mate_rev_manifest.jsonl\")\n", "test_manifest = manifest_data[:]" ] }, { "cell_type": "code", "execution_count": null, "id": "951d8d90", "metadata": {}, "outputs": [], "source": [ "random.shuffle(test_manifest)\n", "for e in test_manifest[:5]:\n", " print(\"-\"*10)\n", " print(e[\"filepath\"])\n", " start_s = round(e[\"start_s\"] + random.random() * (e[\"end_s\"] - e[\"start_s\"]), 1)\n", " end_s = round(np.min([start_s + 30, e[\"end_s\"]]), 1)\n", " print(start_s, end_s)\n", " Audio.from_file(e[\"filepath\"]).play(start_s, end_s)" ] }, { "cell_type": "code", "execution_count": null, "id": "0036a717", "metadata": {}, "outputs": [], "source": [ "# non-aussie accent:\n", "# /mnt/data-ssd-1/data/podcasts/audio/739552/\n", "# tag:blogger.com,1999:blog-8426670135356569597.post-8604780682275250195.mp3" ] }, { "cell_type": "code", "execution_count": 127, "id": "9daf4184", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 4%|▍ | 5/118 [05:32<1:51:06, 59.00s/it] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "b3364829-2339-459e-a5d5-fba70900d919 0.22\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 19%|█▉ | 23/118 [22:12<1:37:48, 61.77s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "c7599326-6bdc-4f6f-8069-ea87253abba2 0.27\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 118/118 [1:37:29<00:00, 49.57s/it] \n" ] } ], "source": [ "# Checking for music\n", "import pickle\n", "from suno_utils.tasks.music import predict_music\n", "\n", "music_p_map = {}\n", "for e in tqdm.tqdm(test_manifest):\n", " audio = Audio.from_file(e[\"filepath\"]).get_slice(e[\"start_s\"], e[\"end_s\"])\n", " music_p = predict_music(audio, frame_duration_ms=1000)\n", " avg_music_p = (music_p >= 0.5).mean()\n", " if avg_music_p > 0.2:\n", " print(e[\"uuid\"], round(avg_music_p, 2))\n", " music_p_map[e[\"uuid\"]] = music_p\n", " with open(\"tmp.pkl\", \"wb\") as f:\n", " pickle.dump(music_p_map, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "a6f07125", "metadata": {}, "outputs": [], "source": [ "# TODO: checking for foreign language using english ASR" ] }, { "cell_type": "code", "execution_count": 138, "id": "be868039", "metadata": {}, "outputs": [], "source": [ "# plt.figure(figsize=(14, 4))\n", "# x = np.linspace(930, 950, len(music_p))\n", "# plt.plot(x, music_p)\n", "# plt.xlabel(\"time (s)\");" ] }, { "cell_type": "code", "execution_count": 137, "id": "0b4f3834", "metadata": {}, "outputs": [], "source": [ "# import plotly.express as px\n", "# x = np.linspace(0, audio.duration_s, len(music_p))\n", "# fig = px.line(x=x, y=music_p, labels={\"x\": \"time (s)\", \"y\": \"music prob\"})\n", "# fig.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "027294f5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1d2fbf53", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a946e675", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bcb1c2cc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a507173e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "085362cd", "metadata": {}, "source": [ "## do rev annotation (round 1)" ] }, { "cell_type": "code", "execution_count": 117, "id": "c45d4b80", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "from suno_utils.utils.text import load_jsonl\n", "from suno_utils.utils.notebook import Audio\n", "import numpy as np\n", "\n", "full_manifest_data = load_jsonl(\"/home/georg/notebooks/datasets/gdata_mate/gdata_mate_rev_manifest.jsonl\")" ] }, { "cell_type": "code", "execution_count": 119, "id": "ece022cb", "metadata": {}, "outputs": [], "source": [ "# respect start/end, use hotwords and accent\n", "exclude_uuids = set([\n", " \"1f2d563c-1c46-457c-badb-66977519e9ef\", # non-aussie accent\n", "])" ] }, { "cell_type": "code", "execution_count": 120, "id": "c9333fdc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "13 episodes with duration: 7.4 hours\n" ] } ], "source": [ "manifest_data = []\n", "seen_podcasts = set()\n", "for m in full_manifest_data:\n", " podcast_id = m[\"filepath\"].split(\"/\")[-2]\n", " if m[\"uuid\"] in exclude_uuids or podcast_id in seen_podcasts:\n", " continue\n", " manifest_data.append(m)\n", " seen_podcasts.add(podcast_id) \n", "duration_h = np.sum([m[\"end_s\"] - m[\"start_s\"] for m in manifest_data]) / 60 / 60\n", "print(\"{} episodes with duration: {} hours\".format(len(manifest_data), round(duration_h, 1)))" ] }, { "cell_type": "code", "execution_count": null, "id": "d5c428a1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "42a14ff2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b16a0dd0", "metadata": {}, "source": [ "### 01 - upload items" ] }, { "cell_type": "code", "execution_count": 7, "id": "cb256e57", "metadata": {}, "outputs": [], "source": [ "PRE_PIPELINE_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/\"" ] }, { "cell_type": "code", "execution_count": 8, "id": "52d44772", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, upload_file" ] }, { "cell_type": "code", "execution_count": 9, "id": "2453089e", "metadata": {}, "outputs": [], "source": [ "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 13, "id": "6f475372", "metadata": {}, "outputs": [], "source": [ "# !mkdir /mnt/data-ssd-1/data/gdata-mate/pre_pipeline/rev_files" ] }, { "cell_type": "code", "execution_count": null, "id": "235bca87", "metadata": {}, "outputs": [], "source": [ "order_items = []\n", "for audio_meta in tqdm.tqdm(manifest_data):\n", " audio = Audio.from_file(audio_meta[\"filepath\"])\n", " audio = audio.get_slice(audio_meta[\"start_s\"], audio_meta[\"end_s\"])\n", " audio = audio.convert(44_100, 2, 1)\n", " wav_audio_filepath = f\"{PRE_PIPELINE_DIR}rev_files/{audio_meta['uuid']}.wav\"\n", " audio.to_wav(wav_audio_filepath)\n", " mp3_audio_filepath = f\"{PRE_PIPELINE_DIR}rev_files/{audio_meta['uuid']}.mp3\"\n", " audio.to_mp3(mp3_audio_filepath)\n", " media_loc = upload_file(AUTH_STR, mp3_audio_filepath, file_ref_str=audio_meta[\"uuid\"])\n", " order_items.append({\n", " \"media_loc\": media_loc,\n", " \"hotwords\": audio_meta[\"hotwords\"],\n", " \"accents\": [\"Australian\"],\n", " \"duration_s\": int(round(audio.duration_s)),\n", " \"uuid\": audio_meta[\"uuid\"],\n", " })" ] }, { "cell_type": "code", "execution_count": 16, "id": "a0ee5f4f", "metadata": {}, "outputs": [], "source": [ "with open(PRE_PIPELINE_DIR + \"01_order_items_p1.json\", \"w\") as f:\n", " json.dump(order_items, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "7fa25492", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "48d46458", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "9131358e", "metadata": {}, "source": [ "### 02 - make order" ] }, { "cell_type": "code", "execution_count": 17, "id": "7ccf6e0a", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, make_order\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 19, "id": "0b300ef5", "metadata": {}, "outputs": [], "source": [ "PRE_PIPELINE_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/\"\n", "\n", "with open(PRE_PIPELINE_DIR + \"01_order_items_p1.json\", \"r\") as f:\n", " order_items = json.load(f)" ] }, { "cell_type": "code", "execution_count": 21, "id": "8318e46f", "metadata": {}, "outputs": [], "source": [ "order_number = make_order(AUTH_STR, order_items, order_ref_str=\"gdata_mate-part_1\")" ] }, { "cell_type": "code", "execution_count": 22, "id": "27549d51", "metadata": {}, "outputs": [], "source": [ "with open(PRE_PIPELINE_DIR + \"02_order_numer_p1.txt\", \"w\") as f:\n", " f.write(order_number)" ] }, { "cell_type": "code", "execution_count": null, "id": "00bf823d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7ad782f1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "210eae71", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f30acc60", "metadata": {}, "source": [ "### 04 - pull transcripts" ] }, { "cell_type": "code", "execution_count": 46, "id": "65101ca4", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 47, "id": "de613d78", "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "PRE_PIPELINE_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/\"\n", "\n", "with open(PRE_PIPELINE_DIR + \"02_order_numer_p1.txt\") as f:\n", " order_number = f.read()" ] }, { "cell_type": "code", "execution_count": 48, "id": "498cc9fc", "metadata": {}, "outputs": [], "source": [ "transcript_metas = get_finished_order(AUTH_STR, order_number, blocking=True)" ] }, { "cell_type": "code", "execution_count": 49, "id": "0c49ce69", "metadata": {}, "outputs": [], "source": [ "for transcript_meta in transcript_metas:\n", " transcript_json = get_transcript(AUTH_STR, transcript_meta[\"rev_id\"])\n", " transcript_filepath = os.path.join(PRE_PIPELINE_DIR, \"rev_files\", f\"{transcript_meta['uuid']}.json\")\n", " with open(transcript_filepath, \"w\") as f:\n", " json.dump(transcript_json, f)" ] }, { "cell_type": "code", "execution_count": 50, "id": "3cd62557", "metadata": {}, "outputs": [], "source": [ "annotation_manifest = [\n", " {\n", " \"audio_filepath\": os.path.join(PRE_PIPELINE_DIR, \"rev_files\", f\"{transcript_meta['uuid']}.wav\"),\n", " \"transcript_filepath\": os.path.join(PRE_PIPELINE_DIR, \"rev_files\", f\"{transcript_meta['uuid']}.json\"),\n", " }\n", " for transcript_meta in transcript_metas\n", "]\n", "assert(all([\n", " os.path.exists(e[\"audio_filepath\"]) and os.path.exists(e[\"transcript_filepath\"]) \n", " for e in annotation_manifest\n", "]))" ] }, { "cell_type": "code", "execution_count": 51, "id": "bce65300", "metadata": {}, "outputs": [], "source": [ "with open(PRE_PIPELINE_DIR + \"03_annotation_manifest_p1.json\", \"w\") as f:\n", " json.dump(annotation_manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "5d868b5d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9e3095b6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "0bddf20d", "metadata": {}, "source": [ "## do rev annotation (round 2)" ] }, { "cell_type": "code", "execution_count": 8, "id": "14457d41", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "from suno_utils.utils.text import load_jsonl\n", "from suno_utils.utils.notebook import Audio\n", "import numpy as np\n", "\n", "full_manifest_data = load_jsonl(\"/home/georg/notebooks/datasets/gdata_mate/gdata_mate_rev_manifest.jsonl\")" ] }, { "cell_type": "code", "execution_count": 9, "id": "7e9994bb", "metadata": {}, "outputs": [], "source": [ "# respect start/end, use hotwords and accent\n", "exclude_uuids = set([\n", " \"1f2d563c-1c46-457c-badb-66977519e9ef\", # non-aussie accent\n", "])\n", "PRE_PIPELINE_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/\"\n", "with open(PRE_PIPELINE_DIR + \"01_order_items_p1.json\", \"r\") as f:\n", " order_items = json.load(f)\n", "for e in order_items:\n", " exclude_uuids.add(e[\"uuid\"])" ] }, { "cell_type": "code", "execution_count": 10, "id": "4f49bc0e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "14 excluded\n", "104 episodes with duration: 57.4 hours\n", "$6884 needed for rev\n" ] } ], "source": [ "n_excluded = 0\n", "manifest_data = []\n", "for m in full_manifest_data:\n", " podcast_id = m[\"filepath\"].split(\"/\")[-2]\n", " if m[\"uuid\"] in exclude_uuids:\n", " n_excluded += 1\n", " continue\n", " manifest_data.append(m) \n", "print(n_excluded, \"excluded\")\n", "duration_h = np.sum([m[\"end_s\"] - m[\"start_s\"] for m in manifest_data]) / 60 / 60\n", "print(\"{} episodes with duration: {} hours\".format(len(manifest_data), round(duration_h, 1)))\n", "print(\"${} needed for rev\".format(int(np.ceil(duration_h * 60 * 2))))" ] }, { "cell_type": "code", "execution_count": null, "id": "2bb68a38", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1e532a3f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f26384ac", "metadata": {}, "source": [ "### 01 - upload items" ] }, { "cell_type": "code", "execution_count": 11, "id": "f679f7e1", "metadata": {}, "outputs": [], "source": [ "PRE_PIPELINE_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/\"" ] }, { "cell_type": "code", "execution_count": 12, "id": "01bb61df", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, upload_file" ] }, { "cell_type": "code", "execution_count": 13, "id": "d7862f3f", "metadata": {}, "outputs": [], "source": [ "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 14, "id": "9fe585b2", "metadata": {}, "outputs": [], "source": [ "# !mkdir /mnt/data-ssd-1/data/gdata-mate/pre_pipeline/rev_files" ] }, { "cell_type": "code", "execution_count": null, "id": "fbd9e7fe", "metadata": {}, "outputs": [], "source": [ "order_items = []\n", "for audio_meta in tqdm.tqdm(manifest_data):\n", " audio = Audio.from_file(audio_meta[\"filepath\"])\n", " audio = audio.get_slice(audio_meta[\"start_s\"], audio_meta[\"end_s\"])\n", " audio = audio.convert(44_100, 2, 1)\n", " wav_audio_filepath = f\"{PRE_PIPELINE_DIR}rev_files/{audio_meta['uuid']}.wav\"\n", " audio.to_wav(wav_audio_filepath)\n", " mp3_audio_filepath = f\"{PRE_PIPELINE_DIR}rev_files/{audio_meta['uuid']}.mp3\"\n", " audio.to_mp3(mp3_audio_filepath)\n", " media_loc = upload_file(AUTH_STR, mp3_audio_filepath, file_ref_str=audio_meta[\"uuid\"])\n", " order_items.append({\n", " \"media_loc\": media_loc,\n", " \"hotwords\": audio_meta[\"hotwords\"],\n", " \"accents\": [\"Australian\"],\n", " \"duration_s\": int(round(audio.duration_s)),\n", " \"uuid\": audio_meta[\"uuid\"],\n", " })" ] }, { "cell_type": "code", "execution_count": 16, "id": "2b14d04e", "metadata": {}, "outputs": [], "source": [ "with open(PRE_PIPELINE_DIR + \"01_order_items_p2.json\", \"w\") as f:\n", " json.dump(order_items, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "8f6eb0e6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "369d26df", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "af609f5e", "metadata": {}, "source": [ "### 02 - make order" ] }, { "cell_type": "code", "execution_count": 31, "id": "bca3d47b", "metadata": {}, "outputs": [], "source": [ "import json\n", "from suno_utils.web.rev import get_auth_string, make_order\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 32, "id": "ded2c236", "metadata": {}, "outputs": [], "source": [ "PRE_PIPELINE_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/\"\n", "\n", "with open(PRE_PIPELINE_DIR + \"01_order_items_p2.json\", \"r\") as f:\n", " order_items = json.load(f)" ] }, { "cell_type": "code", "execution_count": 34, "id": "91724a3e", "metadata": {}, "outputs": [], "source": [ "order_number = make_order(AUTH_STR, order_items, order_ref_str=\"gdata_mate-part_2\")" ] }, { "cell_type": "code", "execution_count": 35, "id": "3449111c", "metadata": {}, "outputs": [], "source": [ "with open(PRE_PIPELINE_DIR + \"02_order_numer_p2.txt\", \"w\") as f:\n", " f.write(order_number)" ] }, { "cell_type": "code", "execution_count": null, "id": "8dee2cd5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fcbd5ba7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8059ae72", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6e0eed5e", "metadata": {}, "source": [ "### 04 - pull transcripts" ] }, { "cell_type": "code", "execution_count": 4, "id": "ecdfb699", "metadata": {}, "outputs": [], "source": [ "import json\n", "import tqdm\n", "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 5, "id": "60c64359", "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "PRE_PIPELINE_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/\"\n", "\n", "with open(PRE_PIPELINE_DIR + \"02_order_numer_p2.txt\") as f:\n", " order_number = f.read()" ] }, { "cell_type": "code", "execution_count": 8, "id": "e70bedfe", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "104 transcripts done.\n" ] } ], "source": [ "transcript_metas = get_finished_order(AUTH_STR, order_number, blocking=True)\n", "print(len(transcript_metas), \"transcripts done.\")" ] }, { "cell_type": "code", "execution_count": 11, "id": "9c060e08", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████| 104/104 [02:25<00:00, 1.40s/it]\n" ] } ], "source": [ "for transcript_meta in tqdm.tqdm(transcript_metas):\n", " transcript_json = get_transcript(AUTH_STR, transcript_meta[\"rev_id\"])\n", " transcript_filepath = os.path.join(PRE_PIPELINE_DIR, \"rev_files\", f\"{transcript_meta['uuid']}.json\")\n", " with open(transcript_filepath, \"w\") as f:\n", " json.dump(transcript_json, f)" ] }, { "cell_type": "code", "execution_count": 12, "id": "ff0fdd25", "metadata": {}, "outputs": [], "source": [ "annotation_manifest = [\n", " {\n", " \"audio_filepath\": os.path.join(PRE_PIPELINE_DIR, \"rev_files\", f\"{transcript_meta['uuid']}.wav\"),\n", " \"transcript_filepath\": os.path.join(PRE_PIPELINE_DIR, \"rev_files\", f\"{transcript_meta['uuid']}.json\"),\n", " }\n", " for transcript_meta in transcript_metas\n", "]\n", "assert(all([\n", " os.path.exists(e[\"audio_filepath\"]) and os.path.exists(e[\"transcript_filepath\"]) \n", " for e in annotation_manifest\n", "]))" ] }, { "cell_type": "code", "execution_count": 13, "id": "c92e7104", "metadata": {}, "outputs": [], "source": [ "with open(PRE_PIPELINE_DIR + \"03_annotation_manifest_p2.json\", \"w\") as f:\n", " json.dump(annotation_manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "cec7e7d0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1449ddcc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "9f92190b", "metadata": {}, "source": [ "### combine manifests" ] }, { "cell_type": "code", "execution_count": 19, "id": "72e30abd", "metadata": {}, "outputs": [], "source": [ "with open(\"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest_p1.json\") as f:\n", " manifest_p1 = json.load(f)\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest_p2.json\") as f:\n", " manifest_p2 = json.load(f) \n", "manifest = manifest_p1 + manifest_p2\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest.json\", \"w\") as f:\n", " json.dump(manifest, f) " ] }, { "cell_type": "code", "execution_count": 17, "id": "521c271e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7e9caed7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e0502844", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "496e57a4", "metadata": {}, "source": [ "## Make dataset" ] }, { "cell_type": "code", "execution_count": null, "id": "e9c787c6", "metadata": {}, "outputs": [], "source": [ "# rm -rf /mnt/data-ssd-1/data/gdata-mate/pipeline/*\n", "# rm -rf /mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts && rm -rf /mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts" ] }, { "cell_type": "code", "execution_count": null, "id": "4ebd702c", "metadata": {}, "outputs": [], "source": [ "# run the pipeline\n", "CUDA_VISIBLE_DEVICES=0 python -m suno_utils.scripts.rev_podcast.run \\\n", " --output_dir=\"/mnt/data-ssd-1/data/gdata-mate/pipeline\" \\\n", " --use_gpu \\\n", " --conda-env=\"mfa\" \\\n", " --conda_sh_filepath=\"/home/georg/anaconda3/etc/profile.d/conda.sh\" \\\n", " --mfa_dictionary_name=\"english\" \\\n", " --mfa_acoustic_model_name=\"english\" \\\n", " --num_cores_mfa=20 \\\n", " --manifest_file=\"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest.json\" \\\n", " --dataset_name=\"gdata_mate\" \\\n", " --sample_rate=44100 \\\n", " --max_cer_thres=0.6 \\\n", " --skip_verify_boundaries \\\n", " --min_silence_gap_s=0.02 \\\n", " --max_segment_duration_s=30.0 \\\n", " --n_align_success_pad=2 \\\n", " --s3_sample_data_prefix=\"s3://suno-static-public/datasets/gdata-mate/sample-data/segments/\"" ] }, { "cell_type": "code", "execution_count": null, "id": "5e5ab743", "metadata": {}, "outputs": [], "source": [ "# Sample stats:\n", "# 2-20s, pad=2: 79.4% / 72.2%\n", "# 2-20s, pad=1: 88.3% / 86.7%\n", "# 2-30s, pad=2: 84.9% / 82.9%\n", "# 2-30s, pad=1: 90.3% / 88.1%" ] }, { "cell_type": "code", "execution_count": null, "id": "c074c976", "metadata": {}, "outputs": [], "source": [ "# INFO:__main__:copying manifest file /mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest.json to /mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/00_segment_manifest.json\n", "# INFO:__main__:Step `0_segmentation` starting...\n", "# WARNING:suno_utils.datasets.rev_podcast.process_data:only 33.7% segment coverage for `/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/rev_files/3045f69a-52ca-4c35-8e83-7f0df9894889.json`.\n", "# WARNING:suno_utils.datasets.rev_podcast.process_data:skipping `/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/rev_files/3045f69a-52ca-4c35-8e83-7f0df9894889.json` due to low coverage and likely mistakes.\n", "# WARNING:suno_utils.datasets.rev_podcast.process_data:skipping `1f10a7e4-86b9-4df8-a445-a487e6044c06` due to segmenting error.\n", "# INFO:suno_utils.datasets.rev_podcast.process_data:84.1% segment coverage after slicing\n", "# INFO:suno_utils.datasets.rev_podcast.pipeline:8063 total segments\n", "# INFO:__main__:Step `0_segmentation` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/01_segment_meta.json`.\n", "# INFO:__main__:Step `0_segmentation` done. Took 9748s, total time: 9748s\n", "# INFO:__main__:Step `1_norm+asr` starting...\n", "# INFO:suno_utils.datasets.rev_podcast.process_data:transcribing 8063 segments using ASR\n", "# Transcribing: 100%|██████████████████████████████████████████████████████| 1008/1008 [03:55<00:00, 4.28it/s]\n", "# INFO:__main__:Step `1_norm+asr` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/02_segment_meta_post_norm_and_asr.json`.\n", "# INFO:__main__:Step `1_norm+asr` done. Took 20444s, total time: 30192s\n", "# INFO:__main__:Step `2_timestamps` starting...\n", "# WARNING:suno_utils.web.mfa:12/8063 failed.\n", "# INFO:suno_utils.datasets.rev_podcast.process_data:12/8063 failed for alignment.\n", "# WARNING:suno_utils.web.mfa:10/8063 failed.\n", "# INFO:suno_utils.datasets.rev_podcast.process_data:10/8063 failed for normalized alignment\n", "# INFO:__main__:Step `2_timestamps` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/03_segment_meta_post_timestamp.json`.\n", "# INFO:__main__:Step `2_timestamps` done. Took 11435s, total time: 41627s\n", "# INFO:__main__:Step `3_distribution` starting...\n", "# INFO:suno_utils.datasets.rev_podcast.pipeline:ignoring 212 of 8063 segments\n", "# INFO:__main__:Step `3_distribution` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/04_segment_meta_distribution.jsonl`.\n", "# INFO:__main__:Step `3_distribution` done. Took 66s, total time: 41693s\n", "# INFO:suno_utils.datasets.rev_podcast.pipeline:81.9% final segment coverage" ] }, { "cell_type": "code", "execution_count": 22, "id": "94e30a8a", "metadata": {}, "outputs": [], "source": [ "import json\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest.json\") as f:\n", " manifest = json.load(f)" ] }, { "cell_type": "code", "execution_count": 23, "id": "b04b32af", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Slice filtering:\n" ] }, { "data": { "text/plain": [ "intro 193\n", "mfa align failure 8\n", "high asr cer 7\n", "intro;mfa align failure 4\n", "dtype: int64" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import json\n", "import pandas as pd\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/03_segment_meta_post_timestamp.json\") as f:\n", " d = json.load(f)\n", "print(\"Slice filtering:\")\n", "pd.Series([e[\"ignore_reason\"] for e in d if e[\"ignore_segment\"]]).value_counts()" ] }, { "cell_type": "code", "execution_count": null, "id": "84308f0b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b35e65ce", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7378dfe4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "912387bd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f825f9f8", "metadata": {}, "source": [ "### (optional) Look at segmenting coverage" ] }, { "cell_type": "code", "execution_count": 39, "id": "65ad996c", "metadata": {}, "outputs": [], "source": [ "import json\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest_p1.json\") as f:\n", " m = json.load(f)\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/03_segment_meta_post_timestamp.json\") as f:\n", " d = json.load(f)" ] }, { "cell_type": "code", "execution_count": 40, "id": "7d2c5840", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-06-10 16:29:35 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-06-10 16:29:35 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-06-10 16:29:35 post_processing:56] Post processing graph was restored from /tmp/prod_nemo_fst/en_tn_post_processing.far.\n", "[NeMo I 2022-06-10 16:29:46 tokenize_and_classify_with_audio:105] ClassifyFst.fst was restored from /tmp/prod_nemo_fst/_cased_en_tn_False_deterministicnemo_custom_norm.tsv.far.\n", "[NeMo I 2022-06-10 16:29:46 verbalize_final:58] VerbalizeFinalFst graph was restored from /tmp/prod_nemo_fst/en_tn_False_deterministic_verbalizer.far.\n", "5a9e6192-e39a-4d7d-946f-96bd99c4bf01: 70%/ 53% cover, 0.9h total, 52% align\n", "3b9a882a-e4ba-444d-837b-f2ce4a804d3b: 86%/ 72% cover, 0.6h total, 74% align\n", "8af98ca8-6148-46b4-b5b0-909dc1e454fe: 88%/ 79% cover, 0.6h total, 64% align\n", "0cebfc24-d177-4bec-a74e-c4b03c1dca25: 89%/ 73% cover, 0.8h total, 72% align\n", "b3f02fca-7f8d-4c1e-b299-fb6cff056992: 92%/ 85% cover, 1.0h total, 72% align\n", "fbf9d6c5-a122-42d4-84cc-15f43dcb72e3: 92%/ 61% cover, 0.3h total, 77% align\n", "2a7c2da5-cb93-441d-8feb-49a6e9a65d5e: 92%/ 82% cover, 0.4h total, 62% align\n", "003542f7-5d11-424d-99c1-e6edd339f3c2: 96%/ 86% cover, 0.6h total, 76% align\n", "340fbfe1-8a8c-4449-9020-b1e1f708d909: 96%/ 82% cover, 0.8h total, 86% align\n", "22b2e88c-13a6-40bb-81f4-2db7a8d83e21: 97%/ 81% cover, 0.2h total, 77% align\n", "924713b2-e33e-4048-a53d-18256ddab3fb: 99%/ 85% cover, 0.3h total, 76% align\n", "a98ca846-47e6-47fa-bbd1-93d65e9ba4d7: 99%/ 88% cover, 0.5h total, 84% align\n", "c49003b5-177c-4bbb-b970-7d5818bf0753: 100%/ 95% cover, 0.4h total, 94% align\n" ] } ], "source": [ "from suno_utils.audio import Audio\n", "from suno_utils.datasets.rev_podcast.process_data import parse_transcript, postprocess_align_dict\n", "\n", "source_duration_map = {\n", " e[\"audio_filepath\"].split(\"/\")[-1].split(\".\")[0]: Audio.from_file(e[\"audio_filepath\"]).duration_s\n", " for e in m\n", "}\n", "\n", "align_frac_map = {}\n", "for e in m:\n", " uid = e[\"audio_filepath\"].split(\"/\")[-1].split(\".\")[0]\n", " gentle_filepath = f\"/mnt/data-ssd-1/data/gdata-mate/pipeline/alignment-gentle/{uid}.json\"\n", " with open(e[\"transcript_filepath\"]) as fi:\n", " transcript_json = json.load(fi)\n", " _, tokens = parse_transcript(transcript_json)\n", " with open(gentle_filepath) as fi:\n", " align_dict = json.load(fi)\n", " aligned_tokens = postprocess_align_dict(align_dict, tokens_with_tags=tokens)\n", " align_frac = np.mean([t.success for t in aligned_tokens])\n", " align_frac_map[uid] = align_frac\n", " \n", "segment_duration_map = {}\n", "for e in d:\n", " if e[\"source_audio_id\"] not in segment_duration_map:\n", " segment_duration_map[e[\"source_audio_id\"]] = 0\n", " segment_duration_map[e[\"source_audio_id\"]] += e[\"duration_s\"]\n", " \n", "segment_duration_final_map = {}\n", "for e in d:\n", " if e[\"ignore_segment\"]:\n", " continue\n", " if e[\"source_audio_id\"] not in segment_duration_final_map:\n", " segment_duration_final_map[e[\"source_audio_id\"]] = 0\n", " segment_duration_final_map[e[\"source_audio_id\"]] += e[\"duration_s\"]\n", "\n", "data = [] \n", "for k, v in source_duration_map.items():\n", " align_frac = align_frac_map[k]\n", " cv = segment_duration_map[k] / v\n", " cv_final = segment_duration_final_map[k] / v\n", " print_line = \"{}: {}%/{}% cover, {}h total, {}% align\".format(\n", " k, \n", " str(int(round(cv * 100))).rjust(3), \n", " str(int(round(cv_final * 100))).rjust(3), \n", " round(v / 60 / 60, 1), \n", " str(int(round(align_frac * 100))).rjust(3),\n", " )\n", " data.append((cv, print_line))\n", " \n", "data = sorted(data, key=lambda x: x[0])\n", "for _, print_line in data:\n", " print(print_line)" ] }, { "cell_type": "code", "execution_count": null, "id": "4eefde89", "metadata": {}, "outputs": [], "source": [ "# 5a9e6192-e39a-4d7d-946f-96bd99c4bf01: 70%/ 53% cover, 0.9h total, 52% align\n", "# 3b9a882a-e4ba-444d-837b-f2ce4a804d3b: 86%/ 72% cover, 0.6h total, 74% align\n", "# 8af98ca8-6148-46b4-b5b0-909dc1e454fe: 88%/ 79% cover, 0.6h total, 64% align\n", "# 0cebfc24-d177-4bec-a74e-c4b03c1dca25: 89%/ 73% cover, 0.8h total, 72% align\n", "# b3f02fca-7f8d-4c1e-b299-fb6cff056992: 92%/ 85% cover, 1.0h total, 72% align\n", "# fbf9d6c5-a122-42d4-84cc-15f43dcb72e3: 92%/ 61% cover, 0.3h total, 77% align\n", "# 2a7c2da5-cb93-441d-8feb-49a6e9a65d5e: 92%/ 82% cover, 0.4h total, 62% align\n", "# 003542f7-5d11-424d-99c1-e6edd339f3c2: 96%/ 86% cover, 0.6h total, 76% align\n", "# 340fbfe1-8a8c-4449-9020-b1e1f708d909: 96%/ 82% cover, 0.8h total, 86% align\n", "# 22b2e88c-13a6-40bb-81f4-2db7a8d83e21: 97%/ 81% cover, 0.2h total, 77% align\n", "# 924713b2-e33e-4048-a53d-18256ddab3fb: 99%/ 85% cover, 0.3h total, 76% align\n", "# a98ca846-47e6-47fa-bbd1-93d65e9ba4d7: 99%/ 88% cover, 0.5h total, 84% align\n", "# c49003b5-177c-4bbb-b970-7d5818bf0753: 100%/ 95% cover, 0.4h total, 94% align" ] }, { "cell_type": "markdown", "id": "324bfced", "metadata": {}, "source": [ "#### look at one in detail" ] }, { "cell_type": "code", "execution_count": 41, "id": "bcae57f0", "metadata": {}, "outputs": [], "source": [ "uid = \"5a9e6192-e39a-4d7d-946f-96bd99c4bf01\"\n", "\n", "audio = Audio.from_file([e[\"audio_filepath\"] for e in m if uid in e[\"audio_filepath\"]][0])" ] }, { "cell_type": "code", "execution_count": 42, "id": "ffeb0e8f", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAADCCAYAAAC7bV8HAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAARqklEQVR4nO3dbYwdV33H8e8PO+YhSUvABoXEqU2VlkaA0mRrUrWKEChgeGEXhYJJBTEiddRiQRFINa1a3IQXpaW8qBQVpdQoIKgTpUVdHt2ogFq1TfCa5gEnTbKkSeM0JSYmAUpJ6uTfF3ec3Bz24e7u3b3r9fcjXe3MmTNzz8y5M/vz3DPeVBWSJEmSnvasUTdAkiRJWm4MyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUmP1qBvQWrt2bW3YsGHUzZAkSdIKd+DAge9W1bqpli27kLxhwwYmJiZG3QxJkiStcEnum26Zwy0kSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoYkiVJkqSGIVmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoYkiVJkqSGIVmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoMFJKTbE5yZ5LJJLumWL49yeEkN3evy7ryc5P8a5KDSW5N8tZh74AkSZI0bKtnq5BkFXAVcBFwCNifZLyqbm+qXltVO5uyHwHvqKq7k7wEOJBkX1U9MoS2S5IkSYtikDvJm4DJqrqnqh4H9gJbB9l4Vd1VVXd30/8FPASsm29jJUmSpKUwSEg+A7i/b/5QV9a6uBtScX2S9e3CJJuANcC359VSSZIkaYkM68G9zwMbquqVwA3ANf0Lk5wOfBp4Z1U92a6cZEeSiSQThw8fHlKTJEmSpPkZJCQ/APTfGT6zK3tKVT1cVY91s58Azj+2LMlPAV8Efr+qbpzqDarq6qoaq6qxdescjSFJkqTRGiQk7wfOTrIxyRpgGzDeX6G7U3zMFuCOrnwN8DngU1V1/XCaLEmSJC2uWf93i6o6mmQnsA9YBeypqoNJrgAmqmoceE+SLcBR4AiwvVv9LcCFwAuTHCvbXlU3D3UvJEmSpCFKVY26Dc8wNjZWExMTo26GJEmSVrgkB6pqbKpl/sU9SZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKkxUEhOsjnJnUkmk+yaYvn2JIeT3Ny9Lutb9pUkjyT5wjAbLkmSJC2W1bNVSLIKuAq4CDgE7E8yXlW3N1WvraqdU2ziT4HnAZcvtLGSJEnSUhjkTvImYLKq7qmqx4G9wNZB36Cq/gH4wTzbJ0mSJC25QULyGcD9ffOHurLWxUluTXJ9kvVDaZ0kSZI0AsN6cO/zwIaqeiVwA3DNXFZOsiPJRJKJw4cPD6lJkiRJ0vwMEpIfAPrvDJ/ZlT2lqh6uqse62U8A58+lEVV1dVWNVdXYunXr5rKqJEmSNHSDhOT9wNlJNiZZA2wDxvsrJDm9b3YLcMfwmihJkiQtrVn/d4uqOppkJ7APWAXsqaqDSa4AJqpqHHhPki3AUeAIsP3Y+kn+CXgZcEqSQ8C7qmrf8HdFkiRJGo5U1ajb8AxjY2M1MTEx6mZIkiRphUtyoKrGplrmX9yTJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhoDheQkm5PcmWQyya4plm9PcjjJzd3rsr5llya5u3tdOszGS5IkSYth9WwVkqwCrgIuAg4B+5OMV9XtTdVrq2pns+4LgA8BY0ABB7p1vzeU1g/LZzNlcX6jlrghM6u+5mTqJk+93mcGqHzJ0xtf6LaX4rgNtE+zuWSO7ew+J4PuX81QbS7HeClNd1zn0qfH9nu6fZxT33V9NMjxmstncaa+GfT9hmVU59Cgqpj2GjkXy2mfpjPV52LYn705tWemc2XA69dCPsvz/Z0z5baWyed8Ifs027Vt2vWafZ/2urQMjtGg1+f5tmu2a+9yM8id5E3AZFXdU1WPA3uBrQNu//XADVV1pAvGNwCb59dUSZIkaWnMeicZOAO4v2/+EPCqKepdnORC4C7gfVV1/zTrntGumGQHsAPgrLPOGqzlJ6Ddu3f3zx03216unrnPA9T/uWFuf27vfTx5er93z1Br6bc19XanrTHU9zue7d69e86f/ePV1J+LqcpGb/Dr16D1ZnuP+W9nOVnIPi3W9ehEMtvndq6/lxdbapZ730neDGyuqsu6+bcDr+ofWpHkhcAPq+qxJJcDb62q1yT5APCcqvpwV+8PgP+tqo9O935jY2M1MTGx4B2bE4dbONxiEA63GGwbDreYk+XwFetMHG4xwHoOt5h9W8vkc+5wi5mdiMMtkhyoqrGplg0y3OIBYH3f/Jld2VOq6uGqeqyb/QRw/qDrSpIkScvNIHeSV9MbQvFaegF3P3BJVR3sq3N6VT3YTb8J+N2quqB7cO8AcF5X9ZvA+VV1ZLr3G8mdZEmSJJ1wZrqTPOuY5Ko6mmQnsA9YBeypqoNJrgAmqmoceE+SLcBR4AiwvVv3SJIr6QVrgCtmCsiSJEnScjDrneSl5p1kSZIkLYWFjkmWJEmSTiiGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKkxUEhOsjnJnUkmk+yaod7FSSrJWDe/Jsknk9yW5JYkrx5OsyVJkqTFs3q2CklWAVcBFwGHgP1Jxqvq9qbeqcB7gZv6in8ToKpekeRFwJeT/FJVPTmsHZAkSZKGbZA7yZuAyaq6p6oeB/YCW6eodyXwEeDHfWXnAF8FqKqHgEeAsYU0WJIkSVpsg4TkM4D7++YPdWVPSXIesL6qvtisewuwJcnqJBuB84H1C2ivJEmStOhmHW4xmyTPAj4GbJ9i8R7gF4AJ4D7gX4AnptjGDmAHwFlnnbXQJkmSJEkLMsid5Ad45t3fM7uyY04FXg58Pcm9wAXAeJKxqjpaVe+rqnOraivwfOCu9g2q6uqqGquqsXXr1s1zVyRJkqThGCQk7wfOTrIxyRpgGzB+bGFVPVpVa6tqQ1VtAG4EtlTVRJLnJTkZIMlFwNH2gT9JkiRpuZl1uEVVHU2yE9gHrAL2VNXBJFcAE1U1PsPqLwL2JXmS3t3ntw+j0ZIkSdJiGmhMclV9CfhSU/aH09R9dd/0vcDPz795kiRJ0tLzL+5JkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSY2BQnKSzUnuTDKZZNcM9S5OUknGuvmTklyT5LYkdyT54LAaLkmSJC2WWUNyklXAVcAbgHOAtyU5Z4p6pwLvBW7qK/514NlV9QrgfODyJBuG0G5JkiRp0QxyJ3kTMFlV91TV48BeYOsU9a4EPgL8uK+sgJOTrAaeCzwOfH9hTZYkSZIW1yAh+Qzg/r75Q13ZU5KcB6yvqi82614P/A/wIPCfwEer6sj8mytJkiQtvgU/uJfkWcDHgPdPsXgT8ATwEmAj8P4kL51iGzuSTCSZOHz48EKbJEmSJC3IICH5AWB93/yZXdkxpwIvB76e5F7gAmC8e3jvEuArVfV/VfUQ8M/AWPsGVXV1VY1V1di6devmtyeSJEnSkAwSkvcDZyfZmGQNsA0YP7awqh6tqrVVtaGqNgA3AluqaoLeEIvXACQ5mV6A/vch74MkSZI0VLOG5Ko6CuwE9gF3ANdV1cEkVyTZMsvqVwGnJDlIL2x/sqpuXWijJUmSpMWUqhp1G55hbGysJiYmRt0MSZIkrXBJDlTVTwwFBv/iniRJkvQTDMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDVSVaNuwzMkOQzcN6K3Xwt8d0TvrR77YPTsg9GzD0bPPhg9+2B5WOn98DNVtW6qBcsuJI9SkomqGht1O05k9sHo2QejZx+Mnn0wevbB8nAi94PDLSRJkqSGIVmSJElqGJKf6epRN0D2wTJgH4yefTB69sHo2QfLwwnbD45JliRJkhreSZYkSZIahmQgyeYkdyaZTLJr1O1ZyZLcm+S2JDcnmejKXpDkhiR3dz9P68qT5M+7frk1yXmjbf3xK8meJA8l+VZf2ZyPe5JLu/p3J7l0FPtyvJqmD3YneaA7H25O8sa+ZR/s+uDOJK/vK/d6NU9J1if5WpLbkxxM8t6u3HNhiczQB54LSyTJc5J8I8ktXR/8UVe+MclN3fG8NsmarvzZ3fxkt3xD37am7JsVo6pO6BewCvg28FJgDXALcM6o27VSX8C9wNqm7E+AXd30LuAj3fQbgS8DAS4Abhp1+4/XF3AhcB7wrfked+AFwD3dz9O66dNGvW/Hy2uaPtgNfGCKuud016JnAxu7a9Qqr1cL7oPTgfO66VOBu7pj7bkw+j7wXFi6PghwSjd9EnBT9/m+DtjWlX8c+K1u+reBj3fT24BrZ+qbUe/fMF/eSYZNwGRV3VNVjwN7ga0jbtOJZitwTTd9DfBrfeWfqp4bgecnOX0E7TvuVdU/Akea4rke99cDN1TVkar6HnADsHnRG79CTNMH09kK7K2qx6rqP4BJetcqr1cLUFUPVtU3u+kfAHcAZ+C5sGRm6IPpeC4MWfd5/mE3e1L3KuA1wPVdeXseHDs/rgdemyRM3zcrhiG5d3Le3zd/iJlPWC1MAX+f5ECSHV3Zi6vqwW76v4EXd9P2zeKa63G3PxbHzu6r/D3HvubHPlh03VfGv0jvLprnwgg0fQCeC0smyaokNwMP0ftH3reBR6rqaFel/3g+day75Y8CL+QE6ANDspbar1bVecAbgHcnubB/YfW+w/G/XFliHveR+QvgZ4FzgQeBPxtpa04QSU4B/gb4nar6fv8yz4WlMUUfeC4soap6oqrOBc6kd/f3ZaNt0fJkSIYHgPV982d2ZVoEVfVA9/Mh4HP0Ts7vHBtG0f18qKtu3yyuuR53+2PIquo73S+rJ4G/5OmvKu2DRZLkJHrh7DNV9bddsefCEpqqDzwXRqOqHgG+BvwyveFEq7tF/cfzqWPdLf9p4GFOgD4wJMN+4Ozuqc419Aalj4+4TStSkpOTnHpsGngd8C16x/vY0+GXAn/XTY8D7+ieML8AeLTvK1Et3FyP+z7gdUlO674KfV1Xpnlqxti/id75AL0+2NY9Vb4ROBv4Bl6vFqQbR/lXwB1V9bG+RZ4LS2S6PvBcWDpJ1iV5fjf9XOAiemPDvwa8uavWngfHzo83A1/tvnGZrm9WjNWzV1nZqupokp30LnCrgD1VdXDEzVqpXgx8rneNZDXw2ar6SpL9wHVJ3gXcB7ylq/8lek+XTwI/At659E1eGZL8NfBqYG2SQ8CHgD9mDse9qo4kuZLeLyeAK6pq0AfRTnjT9MGrk5xL7+v9e4HLAarqYJLrgNuBo8C7q+qJbjter+bvV4C3A7d14zEBfg/PhaU0XR+8zXNhyZwOXJNkFb2bpddV1ReS3A7sTfJh4N/o/WOG7uenk0zSe/h4G8zcNyuFf3FPkiRJajjcQpIkSWoYkiVJkqSGIVmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoYkiVJkqTG/wO0liga0x1W9gAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# plot coverage and look for gaps\n", "plt.figure(figsize=(12, 3))\n", "plt.plot([0, audio.duration_s], [0.5, 0.5], \"-k\", alpha=0.5, linewidth=1)\n", "for e in d:\n", " if uid not in e[\"source_audio_id\"]:\n", " continue\n", " start_s = e[\"source_audio_offset_s\"]\n", " end_s = start_s + e[\"duration_s\"]\n", " c = \"orange\" if e[\"ignore_segment\"] else \"blue\"\n", " plt.plot([start_s, end_s], [0.5, 0.5], \"-\", color=c, linewidth=5)" ] }, { "cell_type": "code", "execution_count": 43, "id": "2c924d25", "metadata": {}, "outputs": [], "source": [ "# can we improve coverage fraction with different params for vad / alignment?" ] }, { "cell_type": "code", "execution_count": 44, "id": "fdfc5ff7", "metadata": {}, "outputs": [], "source": [ "from suno_utils.datasets.rev_podcast.process_data import parse_transcript, postprocess_align_dict\n", "with open([e[\"transcript_filepath\"] for e in m if uid in e[\"transcript_filepath\"]][0]) as fi:\n", " transcript_json = json.load(fi)\n", "_, tokens = parse_transcript(transcript_json)\n", "gentle_filepath = f\"/mnt/data-ssd-1/data/gdata-mate/pipeline/alignment-gentle/{uid}.json\"\n", "with open(gentle_filepath) as fi:\n", " align_dict = json.load(fi)\n", "aligned_tokens = postprocess_align_dict(align_dict, tokens_with_tags=tokens)" ] }, { "cell_type": "code", "execution_count": null, "id": "24ec123b", "metadata": {}, "outputs": [], "source": [ "import portion\n", "from suno_utils.utils.slicer import (\n", " remove_music, get_vad_gaps, get_align_gaps, get_excluded_intervals, get_intervals_from_gaps\n", ")\n", "align_tokens = aligned_tokens\n", "min_gap_s=0.02\n", "vad_level=3\n", "safe_pad=1\n", "min_dur_s=2.0\n", "max_dur_s=30.0\n", "max_unac_vad_tol_s=None\n", "ends_are_safe=True\n", "remove_background_for_vad=False\n", "\n", "if remove_background_for_vad:\n", " clean_audio = remove_music(audio)\n", "else:\n", " clean_audio = audio\n", "\n", "vad_gaps = get_vad_gaps(clean_audio, min_gap_s=min_gap_s, vad_level=vad_level)\n", "\n", "align_gaps = get_align_gaps(\n", " align_tokens, audio.duration_s, min_gap_s=min_gap_s, safe_pad=safe_pad\n", ")\n", "# align_gaps = portion.openclosed(0, audio.duration_s)\n", "\n", "if ends_are_safe:\n", " vad_gaps |= portion.closedopen(0, 0.01)\n", " align_gaps |= portion.closedopen(0, 0.01)\n", " vad_gaps |= portion.openclosed(audio.duration_s - 0.01, audio.duration_s)\n", " align_gaps |= portion.openclosed(audio.duration_s - 0.01, audio.duration_s)\n", "excluded_intervals = get_excluded_intervals(align_tokens, audio.duration_s)\n", "segment_intervals = get_intervals_from_gaps(\n", " vad_gaps,\n", " align_gaps,\n", " excluded_intervals,\n", " min_dur_s=min_dur_s,\n", " max_dur_s=max_dur_s,\n", " max_unac_vad_tol_s=max_unac_vad_tol_s,\n", ")\n", "# get cov fraction\n", "print(\n", " \"{}% coverage fraction\".format(\n", " round(np.sum([(s[1] - s[0]) for s in segment_intervals]) / audio.duration_s * 100, 1)\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "40d95ba5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "46b3988c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0d5cb077", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3646dbaa", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f27ef71c", "metadata": {}, "source": [ "### (optional) Look at asr mismatches" ] }, { "cell_type": "code", "execution_count": 6, "id": "c4491acb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "107 asr boundary issue\n" ] } ], "source": [ "import json\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/03_segment_meta_post_timestamp.json\") as f:\n", " d = json.load(f)\n", "subset = [e for e in d if \"asr boundary issue\" in e[\"ignore_reason\"]]\n", "print(len(subset), \"asr boundary issue\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3896496c", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "from suno_utils.utils.metrics import get_cer\n", "for n, s in enumerate(subset[:5]):\n", " Audio.from_file(s[\"uri\"]).play(compress=True)\n", " t = s[\"transcript_normalized\"][\"text\"]\n", " p = s[\"transcript_asr\"][\"transcript\"]\n", " print(f\"Sample {n}:\")\n", " print(\"NORM:\", t)\n", " print()\n", " print(\"ASR: \", p)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "525d2cd1", "metadata": {}, "outputs": [], "source": [ "# 0: start insert, False\n", "# 2: start insert, False\n", "# 3: end remove, False\n", "# 4: end remove, True\n", "# 5: start insert, True\n", "# 97: start remove, False\n", "# 98: end insert, True\n", "# 100: end remove, False" ] }, { "cell_type": "code", "execution_count": 64, "id": "59c4a2e1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "NORM: you you're bound to target more people if you say you got your happy chicken as you know with hen and stuff and\n", "\n", "ASR: you you're bound to target more people if you say you got your happy chick chicken you know with the their hens and stuff and also\n" ] } ], "source": [ "s = subset[98]\n", "t = s[\"transcript_normalized\"][\"text\"]\n", "p = s[\"transcript_asr\"][\"transcript\"]\n", "print(\"NORM:\", t)\n", "print()\n", "print(\"ASR: \", p)" ] }, { "cell_type": "code", "execution_count": 65, "id": "83d59e6a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[Token(value='know,', type='text', speaker_id=22, start_s=None, end_s=None, metadata=None),\n", " Token(value='with', type='text', speaker_id=22, start_s=None, end_s=None, metadata=None),\n", " Token(value='hen', type='text', speaker_id=22, start_s=None, end_s=None, metadata=None),\n", " Token(value='and', type='text', speaker_id=22, start_s=None, end_s=None, metadata=None),\n", " Token(value='stuff.', type='text', speaker_id=22, start_s=1613.53, end_s=1613.97, metadata=None),\n", " Token(value='And', type='text', speaker_id=22, start_s=1614.38, end_s=1614.5, metadata=None),\n", " Token(value='also', type='text', speaker_id=22, start_s=1615.1, end_s=1615.17, metadata=None),\n", " Token(value='[inaudible]', type='tag', speaker_id=None, start_s=None, end_s=None, metadata=None),\n", " Token(value='um,', type='text', speaker_id=22, start_s=1615.34, end_s=1615.76, metadata=None),\n", " Token(value='the', type='text', speaker_id=22, start_s=1616.47, end_s=1616.62, metadata=None)]" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from suno_utils.datasets.rev_podcast.process_data import parse_transcript, postprocess_align_dict\n", "gentle_filepath = f\"/mnt/data-ssd-1/data/gdata-mate/pipeline/alignment-gentle/{s['source_audio_id']}.json\"\n", "with open(s[\"source_transcript_uri\"]) as fi:\n", " transcript_json = json.load(fi)\n", "_, tokens = parse_transcript(transcript_json)\n", "with open(gentle_filepath) as fi:\n", " align_dict = json.load(fi)\n", "aligned_tokens = postprocess_align_dict(align_dict, tokens_with_tags=tokens)\n", "start_s = s[\"source_audio_offset_s\"]\n", "end_s = s[\"source_audio_offset_s\"] + s[\"duration_s\"]\n", "start_idx = [n for n, t in enumerate(aligned_tokens) if t.success and t.start_s >= start_s][0]\n", "end_idx = [n for n, t in enumerate(aligned_tokens) if t.success and t.end_s <= end_s][-1]\n", "aligned_tokens[end_idx-5:end_idx+5]" ] }, { "cell_type": "code", "execution_count": null, "id": "e604580c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ec70cea6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "daa159ac", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "da74ad5e", "metadata": {}, "source": [ "### (optional) Look at alignment mistakes" ] }, { "cell_type": "code", "execution_count": 58, "id": "0ffe3411", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "4 alignment failures found\n" ] } ], "source": [ "import json\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/03_segment_meta_post_timestamp.json\") as f:\n", " d = json.load(f)\n", "subset = [e for e in d if \"mfa align failure\" in e[\"ignore_reason\"]]\n", "print(len(subset), \"alignment failures found\")" ] }, { "cell_type": "code", "execution_count": null, "id": "665fd9eb", "metadata": {}, "outputs": [], "source": [ "for s in subset[:5]:\n", " print(Tokens.from_dict(s[\"transcript\"][\"tokens\"]).plaintext)\n", " print()" ] }, { "cell_type": "code", "execution_count": 59, "id": "1e941e39", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.mfa import Tokens\n", "input_tokens = Tokens.from_dict(subset[1][\"transcript\"][\"tokens\"])" ] }, { "cell_type": "code", "execution_count": 60, "id": "ccf16f50", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.mfa import _mfa_align_text\n", "transcripts = [input_tokens.plaintext]\n", "word_alignments = _mfa_align_text(\n", " [subset[0][\"uri\"]],\n", " transcripts,\n", " \"/home/georg/anaconda3/etc/profile.d/conda.sh\",\n", " \"mfa\",\n", " num_cores=1,\n", " dictionary_name=\"english\",\n", " acoustic_model_name=\"english\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "24f5af7d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "85d94a30", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "eebd326c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "73bcce8c", "metadata": {}, "source": [ "### (optional) listen to examples" ] }, { "cell_type": "code", "execution_count": 74, "id": "35b8896a", "metadata": {}, "outputs": [], "source": [ "import random\n", "from suno_utils.utils.text import load_jsonl\n", "from suno_utils.utils.notebook import Audio\n", "\n", "DATA_DIR = \"/mnt/data-ssd-1/data/gdata-mate/pipeline/\"\n", "# data = load_jsonl(DATA_DIR + \"artifacts/04_segment_meta_distribution.jsonl\")\n", "with open(DATA_DIR + \"artifacts/03_segment_meta_post_timestamp.json\") as f:\n", " data = json.load(f)\n", "random.seed(6006)\n", "random.shuffle(data)" ] }, { "cell_type": "code", "execution_count": 75, "id": "25edc766", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5f2654d1-651f-416d-836b-da174275e2d5\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: honored what he was feeling and wanted to look -- Hmm. Um, man, he just, he looked amazing. He felt amazing. And he was getting out there. And now again, I'm not keeping track of [laughter] people's dates,\n", "\n", "TRUE: honored what he was feeling and wanted to look hmm um man he just he looked amazing he felt amazing and he was getting out there and now again i'm not keeping track of people's dates\n", "\n", "PRED: honored what he was feeling and wanted to look um man he just he looked amazing he felt amazing and he was getting out there and now again i'm not keeping track of people's daates\n", "----------\n", "7d566a03-938c-493b-90b4-92f1096db63a\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: And it's gonna take you that little bit longer, to get your fingertips to point to the bottom of the pool, which is what we need to happen as we go through the catch. The other thing that people sometimes do here is they go too deep. Some people will be well below their shoulder or underarm depth with their hand, and again, it just increases the drag and it doesn't give you as much room, to set up the catch. So, if you can get the starting catch position right, then think of that as when one arm's out in front, you've rotated on your side a little bit, and your fingers, ideally, should be below wrist, wrist\n", "\n", "TRUE: and it's gonna take you that little bit longer to get your fingertips to point to the bottom of the pool which is what we need to happen as we go through the catch the other thing that people sometimes do here is they go too deep some people will be well below their shoulder or underarm depth with their hand and again it just increases the drag and it doesn't give you as much room to set up the catch so if you can get the starting catch position right then think of that as when one arm's out in front you've rotated on your side a little bit and your fingers ideally should be below wrist wrist\n", "\n", "PRED: it's going to take that a little bit longer to get your finger tips to point to the bottom of the pool which is what we need to happen as we go through the catch the other thing that people sometimes do here is they go to deep some people will be well below their shoulder or under arm depth with their hand and again it just increases the drag and it doesn't give you as much room to set up the catch so if you can get the starting catch position right and think of that as when one arm's out in front you're rotated on your side a little bit and your fingers ideally should be below wrist wrist\n", "----------\n", "d0da91e6-5e09-43d8-aa3b-ce50027b5881\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: Hello everyone,\n", "\n", "TRUE: hello everyone\n", "\n", "PRED: hello everyone\n", "----------\n", "26acb391-e3c7-4ccf-9ee1-e3a57b471d0e\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: an elite background. And I wanna ask you how many training sessions you were doing and stuff like that, but that you are, you're taking those lessons and applying them to your, your weekend warrior to, to your average, Joe. And that's such a big part of this podcast is, I firmly believe that the training principles that applied to you as an elite swimmer, apply in a diluted fashion, obviously, for time and cost reasons and all those things. But they apply to,\n", "\n", "TRUE: an elite background and i wanna ask you how many training sessions you were doing and stuff like that but that you are you're taking those lessons and applying them to your your weekend warrior to to your average joe and that's such a big part of this podcast is i firmly believe that the training principles that applied to you as an elite swimmer apply in a diluted fashion obviously for time and cost reasons and all those things but they apply to\n", "\n", "PRED: an elite background and i want to ask you how many training sessions you are doing and stuff like that but that you're you're taking those lessons and applying them to your your week-end warrior to your average joe and that's such a big part of this podcast is i firmly believe that the training principles that applied to you as an elite swimmer apply in a diluted fashion obviously for time and cost reasons and all those things but they apply to\n", "----------\n", "7f249675-aa3b-4461-8f2b-07a4f76c7036\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: aids it perfectly. Mm-hmm. And that is a song, I don't know if you know -- i -- if you can tell me where it's from, then I'll, I'll love it.\n", "\n", "TRUE: aids it perfectly mm hmm and that is a song i don't know if you know i if you can tell me where it's from then i'll i'll love it\n", "\n", "PRED: aids it perfectly mm and that is a song i don't know if you know if you can tell me where it's from then i'll i'll love it\n", "----------\n", "c66e69f5-52e5-487a-a96c-b9ab91b3f559\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: Yes, actually I have been in one. Um, I've only gone one time. Uh, it was -- Uh, was it last summer? Uh, I was with a, a bunch of friends and we were walking through Akihabara and I was like, \"All right -- \" Like this is one of those things in Japan that you just have to do. Like you have to -- I, I try to experience as much as I can while I'm here, 'cause I don't know how long I'll be in Japan. Um, but I like to take advantage of many different things and so I like to\n", "\n", "TRUE: yes actually i have been in one um i've only gone one time uh it was uh was it last summer uh i was with a a bunch of friends and we were walking through akihabara and i was like all right like this is one of those things in japan that you just have to do like you have to i i try to experience as much as i can while i'm here cause i don't know how long i'll be in japan um but i like to take advantage of many different things and so i like to\n", "\n", "PRED: yes actually i have been in one um i've only gone one time uh it was oh what was it last summer i was with a a bunch of friends and we were walking through ockiabita and i was like all right like this is one of those things in japan that you just have to do like you have to i i try to experience as much as i can while i'm here because i don't know how long i'll be in japan uh but i like to take advantage of many different things and so i like to\n", "----------\n", "b77b54af-47dd-4e2b-a19c-b44b0f1177b8\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: jealousy in business, and why no one talks about that, you know? And it's all got to be sunshine and rainbows and unicorns like, \"Oh, you know, I'm happy. I'm just lifting you up, bloody blah, blah, blah.\" But we actually talked about, you know, how it's can, you know, you can be jealous [laughter] of some people's businesses and that's completely fricking okay. [laughter] So, um, so anyway, I'll go into that in a -- in another, in\n", "\n", "TRUE: jealousy in business and why no one talks about that you know and it's all got to be sunshine and rainbows and unicorns like oh you know i'm happy i'm just lifting you up bloody blah blah blah but we actually talked about you know how it's can you know you can be jealous of some people's businesses and that's completely fricking okay so um so anyway i'll go into that in a in another in\n", "\n", "PRED: genoese and business and why no one talks about that you know and it's all got to be sunshine and rainbows and unicorns like oh you know i'm happy ands lifting you up blah you by blah but we actually talked about you know how it's can you know you can be jealous of some people's businesses and that's completely fricking okay so um so anyway i'll go into that in a in another in\n", "----------\n", "0faa72a8-1738-4f0b-a0ca-6affa66c8bf5\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: please go ahead and subscribe to the podcast so you know when episodes are dropped every two weeks, which is what I do, and um, when a new episode is live, so that you can listen in and get these truth bombs show [laughter] straight to your ear holes, um, each and every two weeks. So, um, I have changed the schedule going from weekly down to, um, every two weeks, because I just need to be able to\n", "\n", "TRUE: please go ahead and subscribe to the podcast so you know when episodes are dropped every two weeks which is what i do and um when a new episode is live so that you can listen in and get these truth bombs show straight to your ear holes um each and every two weeks so um i have changed the schedule going from weekly down to um every two weeks because i just need to be able to\n", "\n", "PRED: please go ahead and subscribe to the podcast so you know when episodes are dropped every two weeks which is what i do and when a new episode is live so that you can listen in and get these truth bombs show straight to your earholes each and every two weeks so m i have changed the schedule going from weekly down to um every two weeks because i just need to be able to\n", "----------\n", "f1c5d0db-3220-4dd3-af25-f7260d4820be\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: I've been, you know, wanting to go to Nashville for so long and been planning this specific trip for, you know, over a year planning every detail. Um, so yeah, it's hard, but I definitely don't wanna complain because I know, you know, some musicians have lost their entire income, a lot of musicians have lost their entire income. So yeah, I definitely don't wanna complain too much because I'm definitely in a better, better position than them, um, because I do have another job. Um,\n", "\n", "TRUE: i've been you know wanting to go to nashville for so long and been planning this specific trip for you know over a year planning every detail um so yeah it's hard but i definitely don't wanna complain because i know you know some musicians have lost their entire income a lot of musicians have lost their entire income so yeah i definitely don't wanna complain too much because i'm definitely in a better better position than them um because i do have another job um\n", "\n", "PRED: i've been you know wanting to go to nashville for so long and been planning this specific trip for you know over a year planning every detail um so yeah it's hard but i definitely don't want to complain because i know you know some musicians have lost their entire income a lot of musicians have lost their entire income so yeah i definitely don't want to complain too much because i'm definitely in a better better position than them um because i do have another job um\n", "----------\n", "135b7bce-84f8-4f55-b9f3-3e5e79197d91\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: So let's talk about some Japanese candy.\n", "\n", "TRUE: so let's talk about some japanese candy\n", "\n", "PRED: so let's talk about some japanese candy\n", "----------\n" ] } ], "source": [ "for e in data[:10]:\n", " print(e[\"id\"])\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"TRUE:\", e[\"transcript\"][\"text\"])\n", " print()\n", " print(\"TRUE:\", e[\"transcript_normalized\"][\"text\"])\n", " print()\n", " print(\"PRED:\", e[\"transcript_asr\"][\"transcript\"])\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 131, "id": "7449a888", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████| 8063/8063 [00:01<00:00, 7216.09it/s]\n" ] } ], "source": [ "import tqdm\n", "import pandas as pd\n", "from suno_utils.utils.metrics import get_wer, get_cer\n", "\n", "m_data = []\n", "for e in tqdm.tqdm(data):\n", " wer = get_wer(e[\"transcript_normalized\"][\"text\"], e[\"transcript_asr\"][\"transcript\"])\n", " cer = get_cer(e[\"transcript_normalized\"][\"text\"], e[\"transcript_asr\"][\"transcript\"])\n", " m_data.append((\n", " e[\"id\"],\n", " e[\"ignore_segment\"],\n", " e[\"transcript_asr\"][\"transcript\"],\n", " e[\"transcript_normalized\"][\"text\"],\n", " e[\"transcript_asr\"][\"transcript\"],\n", " e[\"uri\"],\n", " wer,\n", " cer,\n", " ))\n", "metrics_df = pd.DataFrame(m_data, columns=[\"id\", \"ignore\", \"pred\", \"true\", \"raw_true\", \"uri\", \"wer\", \"cer\"])\n", "metrics_df = metrics_df.sort_values(by=\"cer\", ascending=False).reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 141, "id": "abe515e5", "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25 samples\n", "----------\n", "0f617ba1-d2ee-4d77-a9a5-ea44fd3c5777 - 0.519 - 0.6\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: push the accelerator and um\n", "\n", "PRED: pushis ot and um\n", "----------\n", "07ba5e66-6738-4aee-855d-c6fee0f014fd - 0.486 - 0.571\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: and until next time gorgeous ones see\n", "\n", "PRED: and until next time we'lljust on see you later\n", "----------\n", "d3f8f2c2-1088-496f-88ea-188bd9255f9b - 0.4 - 1.0\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: chile\n", "\n", "PRED: chilli\n", "----------\n", "e6ac1df7-6705-450f-9852-bb004655adfd - 0.342 - 0.75\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: say whatever you want but i don't care\n", "\n", "PRED: say what everyone the islan't care\n", "----------\n", "191dae5e-73eb-407b-a424-6cd595c3c063 - 0.286 - 0.667\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: no excuse okay\n", "\n", "PRED: no ice ckuse okay\n", "----------\n", "b8bafd60-a1b7-41ae-b270-859ab5b60917 - 0.28 - 0.4\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: being healthy and fit and\n", "\n", "PRED: stay healthy and stit and\n", "----------\n", "16423e7c-521c-4af1-835e-d3cae94c5b8f - 0.227 - 0.25\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: for tokyo munchies and\n", "\n", "PRED: for tokyo mytids and\n", "----------\n", "4c33a107-391a-41c7-b4cd-1cf1f0001115 - 0.189 - 0.286\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: she was holding her breath eight yeah\n", "\n", "PRED: she was holding a breath eight n\n", "----------\n", "ec304607-3a1a-4907-9588-a7a9d697a822 - 0.167 - 0.333\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: the strongest boats and and um\n", "\n", "PRED: the strongest of otes and and um\n", "----------\n", "0f3d3d66-dc92-40a1-a905-a901953d0f96 - 0.161 - 0.143\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "TRUE: in japan top your top five well\n", "\n", "PRED: in japan top your top five\n", "----------\n" ] } ], "source": [ "_df = metrics_df\n", "_df = _df[~_df[\"ignore\"]]\n", "_df = _df[(_df[\"true\"].str.len() <= 40) | (_df[\"pred\"].str.len() <= 40)]\n", "_df = _df[_df[\"cer\"] > 0]\n", "print(_df.shape[0], \"samples\")\n", "print(\"-\"*10)\n", "for _, row in _df.head(10).iterrows():\n", " print(row[\"id\"], \"-\", round(row[\"cer\"], 3), \"-\", round(row[\"wer\"], 3))\n", " Audio.from_file(row[\"uri\"]).play()\n", " print(\"TRUE:\", row[\"true\"])\n", " print()\n", " print(\"PRED:\", row[\"pred\"])\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 79, "id": "8925db80", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0: two by seven, one by eight, two by nine, one by 10. And then I'd finished up with, uh, three 12 hour swims. Um, one consecutively.\n", "\n", "Brenton Ford: Wow. And, um, uh, and how did you, um, h -- h -- how did you, sort of, manage your, your emotions and your energy during that, that time? Because, I mean, when I train -- I trained for a 20 kilometers swim, which is nothing compared to 125 kilometers, and, uh, like I remember being, um, a pretty,\n", "\n", "----------\n", "TEXT: two by seven, one by eight, two by nine, one by 10. And then I'd finished up with, uh, three 12 hour swims. Um, one consecutively. Wow. And, um, uh, and how did you, um, h -- h -- how did you, sort of, manage your, your emotions and your energy during that, that time? Because, I mean, when I train -- I trained for a 20 kilometers swim, which is nothing compared to 125 kilometers, and, uh, like I remember being, um, a pretty,\n", "\n", "NORM: two by seven one by eight two by nine one by ten and then i'd finished up with uh three twelve hour swims um one consecutively wow and um uh and how did you um h h how did you sort of manage your your emotions and your energy during that that time because i mean when i train i trained for a twenty kilometers swim which is nothing compared to one hundred and twenty five kilometers and uh like i remember being um a pretty\n" ] } ], "source": [ "from suno_utils.audio.data_model import Tokens\n", "for e in data:\n", " if \"--\" not in e[\"transcript\"][\"text\"]:\n", " continue\n", " if not re.search(\"[0-9]\", e[\"transcript\"][\"text\"]):\n", " continue\n", " tokens = Tokens.from_dict(e[\"transcript\"][\"tokens\"])\n", " n_speakers = len(set([t.speaker_id for t in tokens.tokens if t.speaker_id is not None]))\n", " if n_speakers < 2:\n", " continue\n", " for ee in tokens.speaker_turns:\n", " print(\"{}: {}\".format(ee[\"speaker_id\"], ee[\"text\"]))\n", " print()\n", " print(\"-\"*10)\n", " print(\"TEXT:\", e[\"transcript\"][\"text\"])\n", " print()\n", " print(\"NORM:\", e[\"transcript_normalized\"][\"text\"])\n", " break" ] }, { "cell_type": "code", "execution_count": null, "id": "5cb20e2d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f8d1b394", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "a0e9553f", "metadata": {}, "source": [ "### Nvidia dataset viewer" ] }, { "cell_type": "code", "execution_count": null, "id": "f3e314d1", "metadata": {}, "outputs": [], "source": [ "python /home/georg/code/NeMo/tools/speech_data_explorer/data_explorer.py \\\n", " /mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/nvidia_dataset_viewer.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "e909f63a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "13eed023", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b17a6cb8", "metadata": {}, "source": [ "## Make final TAR" ] }, { "cell_type": "code", "execution_count": 122, "id": "e5e803c4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "53.0 hours\n" ] } ], "source": [ "import json\n", "from suno_utils.datasets.rev_podcast.pipeline import write_sample_data_tar\n", "with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/04_segment_meta_distribution.jsonl\") as f:\n", " segments = [json.loads(l) for l in f.read().strip().split(\"\\n\")]\n", "print(round(np.sum([s[\"duration_s\"] for s in segments]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 151, "id": "45717a20", "metadata": {}, "outputs": [], "source": [ "import tempfile\n", "from suno_utils.datasets.rev_podcast.pipeline import process_data\n", "def write_data_tar(main_dir, sample_segments, tar_file_path, dataset_name):\n", " sample_segments = sample_segments[:] # shitty copy\n", " segment_uris = [os.path.join(main_dir + sm[\"uri\"]) for sm in sample_segments]\n", " with tempfile.TemporaryDirectory() as temp_dir:\n", " sample_metadata_filepath = os.path.join(temp_dir, \"segment_meta.jsonl\")\n", " with open(sample_metadata_filepath, \"w\") as fi:\n", " for sm in sample_segments:\n", " fi.write(json.dumps(sm) + \"\\n\")\n", " process_data.write_sample_data_tar(\n", " tar_file_path,\n", " segment_uris,\n", " sample_metadata_filepath,\n", " outer_dir_name=dataset_name,\n", " )" ] }, { "cell_type": "code", "execution_count": 160, "id": "0e91924e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "13.5 Gb\n" ] } ], "source": [ "import copy\n", "segment_metas = [copy.deepcopy(s) for s in segments]#[:5]]\n", "tar_file_path = '/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/data.tar.gz'\n", "dataset_name = \"gdata_mate\"\n", "main_dir = '/mnt/data-ssd-1/data/gdata-mate/pipeline/'\n", "write_data_tar(main_dir, segment_metas, tar_file_path, dataset_name)\n", "print(round(os.path. getsize(tar_file_path) / 1e9, 1), \"Gb\")" ] }, { "cell_type": "code", "execution_count": 162, "id": "6e63f16f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "-rw-rw-r-- 1 georg georg 13G Jun 13 15:34 /mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/data.tar.gz\r\n" ] } ], "source": [ "!ls -lah /mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/data.tar.gz" ] }, { "cell_type": "code", "execution_count": null, "id": "8e40007e", "metadata": {}, "outputs": [], "source": [ "## move to s3\n", "# aws s3 cp --recursive \\\n", "# /mnt/data-ssd-1/data/gdata-mate \\\n", "# s3://suno-data/datasets/gdata-mate" ] }, { "cell_type": "code", "execution_count": null, "id": "f499d8a1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "90979a75", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "dfa08c17", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 37, "id": "b25b3b3e", "metadata": {}, "outputs": [], "source": [ "with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/04_segment_meta_distribution.jsonl\") as f:\n", " d = json.loads(f.read().strip().split(\"\\n\")[0])" ] }, { "cell_type": "code", "execution_count": 85, "id": "05e89012", "metadata": {}, "outputs": [], "source": [ "with open(\"/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/01_segment_meta.json\") as f:\n", " d = json.load(f)" ] }, { "cell_type": "code", "execution_count": null, "id": "b07869af", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a6a50590", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ff124b07", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bb4bf468", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }