{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "bf70f34f", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import json\n", "import tqdm\n", "import uuid\n", "import funcy\n", "import numpy as np\n", "from suno_utils.audio import Audio, Token, Tokens\n", "from suno_utils.audio.conversion import get_audio_properties, play_audio\n", "from suno_utils.utils.numbers import safe_round\n", "from suno_utils.utils.text import normalize_whitespace\n", "from suno_utils.web.harvest import get_file_ext, get_filename\n", "\n", "BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/speechly/gymclass-30h\"\n", "RAW_DATA_DIR = os.path.join(BASE_DIR, \"raw_data\")\n", "RAW_AUDIO_DIR = os.path.join(RAW_DATA_DIR, \"audio\")\n", "\n", "AUDIO_DIR = os.path.join(BASE_DIR, \"audio\")\n", "TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\")\n", "TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n", "FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\")\n", "\n", "SEGMENT_DATE_DIR = \"2022_09_23\"\n", "SEGMENTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"segments\")\n", "ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"artifacts\")" ] }, { "cell_type": "code", "execution_count": 2, "id": "c769f21a", "metadata": {}, "outputs": [], "source": [ "os.makedirs(AUDIO_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_AUDIO_DIR, exist_ok=True)\n", "os.makedirs(FROM_REV_DIR, exist_ok=True)\n", "os.makedirs(SEGMENTS_DIR, exist_ok=True)\n", "os.makedirs(ARTIFACTS_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 3, "id": "c3cd79ed", "metadata": {}, "outputs": [], "source": [ "GLOBAL_HOTWORDS = [\n", " 'beta lobby',\n", " 'beta physics',\n", " 'boom discord ball',\n", " 'bozo',\n", " 'carpet muncher',\n", " 'coke butter cheese',\n", " 'comp rules',\n", " 'dadass',\n", " 'deadass',\n", " 'dribble',\n", " 'dunk competition',\n", " 'dunk dink dunk',\n", " 'dunk on dunk',\n", " 'faggy',\n", " 'freaking weirdo',\n", " 'fuckulus',\n", " 'gorilla tag mid',\n", " 'green bean',\n", " 'high glitch',\n", " 'invisible wall',\n", " 'red skin',\n", " 'rim',\n", " 'watermelon munchers',\n", " 'you sped',\n", "]" ] }, { "cell_type": "markdown", "id": "3bf11bab", "metadata": {}, "source": [ "## Prepare rev data" ] }, { "cell_type": "code", "execution_count": 6, "id": "965eb6c8", "metadata": {}, "outputs": [], "source": [ "SAMPLE_RATE = 16_000" ] }, { "cell_type": "code", "execution_count": 8, "id": "9ff97b1c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 14573/14573 [00:02<00:00, 4980.86it/s]\n" ] } ], "source": [ "# some basic checks (since uid is in filename)\n", "uids = []\n", "for fn in os.listdir(RAW_AUDIO_DIR):\n", " if get_file_ext(fn) != \"wav\":\n", " print(\"found unknown file:\", fn)\n", " continue\n", " uid = get_filename(fn)\n", " assert(len(uid) == 36)\n", " uids.append(uid)\n", "assert(len(uids) == len(set(uids)))\n", "\n", "# save in dir with known sample rate as wavs\n", "filepaths_info = []\n", "for fn in tqdm.tqdm(os.listdir(RAW_AUDIO_DIR)):\n", " uid = get_filename(fn)\n", " from_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " to_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " audio = Audio.from_file(os.path.join(RAW_AUDIO_DIR, fn), sample_rate=SAMPLE_RATE, byte_width=2)\n", " to_filepath = os.path.join(AUDIO_DIR, uid + \".wav\")\n", " audio.to_wav(to_filepath)\n", " filepaths_info.append((uid, from_fp, to_filepath))\n", "# should be 14573" ] }, { "cell_type": "code", "execution_count": 9, "id": "deb0bebc", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"raw_data_manifest.json\"), \"w\") as f:\n", " json.dump(filepaths_info, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "291b0f30", "metadata": {}, "outputs": [], "source": [ "# This audio has no sub-sampling, otherwise use: http://localhost:8040/notebooks/customers/level/pipeline.ipynb" ] }, { "cell_type": "code", "execution_count": 10, "id": "4da818da", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "45.1 mins of audio (16.1% overhead)\n", "45.1 mins of audio (17.2% overhead)\n", "45.0 mins of audio (16.0% overhead)\n", "45.1 mins of audio (16.3% overhead)\n", "45.1 mins of audio (17.9% overhead)\n", "45.2 mins of audio (16.9% overhead)\n", "45.1 mins of audio (17.4% overhead)\n", "45.1 mins of audio (17.2% overhead)\n", "45.0 mins of audio (17.2% overhead)\n", "45.0 mins of audio (17.2% overhead)\n", "45.1 mins of audio (15.7% overhead)\n", "45.0 mins of audio (17.1% overhead)\n", "45.2 mins of audio (17.2% overhead)\n", "45.1 mins of audio (16.5% overhead)\n", "45.1 mins of audio (18.4% overhead)\n", "45.7 mins of audio (17.8% overhead)\n", "45.1 mins of audio (17.2% overhead)\n", "45.1 mins of audio (16.6% overhead)\n", "45.1 mins of audio (17.1% overhead)\n", "45.0 mins of audio (16.7% overhead)\n", "45.3 mins of audio (16.6% overhead)\n", "45.1 mins of audio (16.9% overhead)\n", "45.0 mins of audio (16.5% overhead)\n", "45.1 mins of audio (18.0% overhead)\n", "45.0 mins of audio (16.9% overhead)\n", "45.3 mins of audio (15.9% overhead)\n", "45.0 mins of audio (16.2% overhead)\n", "45.7 mins of audio (16.8% overhead)\n", "45.0 mins of audio (15.0% overhead)\n", "45.0 mins of audio (17.9% overhead)\n", "45.5 mins of audio (17.4% overhead)\n", "45.0 mins of audio (16.8% overhead)\n", "45.2 mins of audio (18.4% overhead)\n", "45.0 mins of audio (16.7% overhead)\n", "45.0 mins of audio (16.7% overhead)\n", "45.2 mins of audio (17.1% overhead)\n", "45.1 mins of audio (17.0% overhead)\n", "45.0 mins of audio (17.6% overhead)\n", "45.0 mins of audio (15.9% overhead)\n", "45.0 mins of audio (17.2% overhead)\n", "45.0 mins of audio (17.8% overhead)\n", "45.2 mins of audio (16.7% overhead)\n", "45.1 mins of audio (16.9% overhead)\n", "45.2 mins of audio (16.5% overhead)\n", "45.1 mins of audio (17.0% overhead)\n", "45.3 mins of audio (15.0% overhead)\n", "45.0 mins of audio (17.6% overhead)\n", "26.3 mins of audio (16.8% overhead)\n" ] } ], "source": [ "MIN_REV_FILE_DURATION_S = 45 * 60\n", "INDICATOR_AUDIO_DIR = \"/mnt/data-ssd-1/data/custom/segment_indicators\"\n", "SILENCE_DURATION_S = 0.25\n", "\n", "audio_rev_metas = []\n", "tmp_segments_audio = []\n", "tmp_segments_meta = []\n", "tmp_n = 0\n", "for n_step, (uid, _, filepath) in enumerate(filepaths_info):\n", " # add indicator\n", " indicator_str = str(tmp_n).zfill(3)\n", " tmp_indicator_audios = []\n", " for c in indicator_str:\n", " tmp_indicator_audios.append(\n", " Audio.from_file(\n", " os.path.join(INDICATOR_AUDIO_DIR, f\"{c}_fast.wav\"), \n", " sample_rate=SAMPLE_RATE, \n", " byte_width=2,\n", " )\n", " )\n", " indicator_audio = Audio.concatenate(tmp_indicator_audios)\n", " tmp_segments_audio.append(indicator_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"indicator\",\n", " \"duration_s\": indicator_audio.duration_s,\n", " \"indicator_str\": indicator_str,\n", " })\n", " # add silence\n", " if SILENCE_DURATION_S > 0:\n", " silence_audio = Audio.from_array(\n", " np.zeros(int(SAMPLE_RATE * SILENCE_DURATION_S), dtype=np.int16), SAMPLE_RATE\n", " )\n", " tmp_segments_audio.append(silence_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"silence\",\n", " \"duration_s\": SILENCE_DURATION_S,\n", " })\n", " # add audio\n", " audio = Audio.from_file(filepath, sample_rate=SAMPLE_RATE, byte_width=2)\n", "# audio = remove_music(audio, wet=1.0, model_name=\"mdx_extra_q\")\n", " tmp_segments_audio.append(audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"speech\",\n", " \"duration_s\": audio.duration_s,\n", " \"orginial_audio_offset_s\": 0,\n", " \"orginial_audio_uid\": uid,\n", " })\n", " # add silence\n", " if SILENCE_DURATION_S > 0:\n", " silence_audio = Audio.from_array(\n", " np.zeros(int(SAMPLE_RATE * SILENCE_DURATION_S), dtype=np.int16), SAMPLE_RATE\n", " )\n", " tmp_segments_audio.append(silence_audio)\n", " tmp_segments_meta.append({\n", " \"segment_number\": tmp_n,\n", " \"type\": \"silence\",\n", " \"duration_s\": SILENCE_DURATION_S,\n", " })\n", " # prep for next step and decide if we make new file\n", " tmp_n += 1\n", " running_duration_s = np.sum([e[\"duration_s\"] for e in tmp_segments_meta])\n", " if (\n", " (running_duration_s >= MIN_REV_FILE_DURATION_S) or \n", " (n_step == len(filepaths_info) - 1 and running_duration_s > 0)\n", " ):\n", " audio_rev = Audio.concatenate(tmp_segments_audio)\n", " running_speech_duration_s = np.sum([e[\"duration_s\"] for e in tmp_segments_meta if e[\"type\"] == \"speech\"])\n", " print(\"{} mins of audio ({}% overhead)\".format(\n", " round(running_duration_s / 60, 1),\n", " round((1 - running_speech_duration_s / running_duration_s) * 100, 1)\n", " )) \n", " uid = str(uuid.uuid4())\n", " rev_audio_filepath = os.path.join(TO_REV_AUDIO_DIR, f\"{uid}.mp3\")\n", " audio_rev.to_mp3(rev_audio_filepath)\n", " audio_rev_metas.append({\n", " \"uid\": uid,\n", " \"audio_filepath\": rev_audio_filepath,\n", " \"segments_meta\": tmp_segments_meta,\n", " \"duration_s\": audio_rev.duration_s,\n", " })\n", " tmp_segments_audio = []\n", " tmp_segments_meta = []\n", " tmp_n = 0" ] }, { "cell_type": "code", "execution_count": 11, "id": "4d16d48e", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\"), \"w\") as f:\n", " json.dump(audio_rev_metas, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "a453909d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "5fc18307", "metadata": {}, "source": [ "## (optional) investigate" ] }, { "cell_type": "code", "execution_count": 12, "id": "dc409c9d", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 16, "id": "ea85a5f2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "14573 transcripts\n" ] }, { "data": { "text/plain": [ "[('04619b14-6652-4516-be14-0909d5f6654e',\n", " 'so i get this bomb and run it so hard in the life'),\n", " ('026d2cce-da91-4d2b-8023-82c71feb966f', 'hey come')]" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "with open(os.path.join(RAW_DATA_DIR, \"utterances.tsv\")) as f:\n", " transcripts = [e.split(\"\\t\") for e in f.read().strip().split(\"\\n\")]\n", "assert(all([len(e) == 2 for e in transcripts]))\n", "transcripts = [(uid, text.lower()) for uid, text in transcripts]\n", "print(len(transcripts), \"transcripts\")\n", "transcripts[:2]" ] }, { "cell_type": "code", "execution_count": 23, "id": "851f3ce9", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio.conversion import play_audio" ] }, { "cell_type": "code", "execution_count": 24, "id": "3f33f396", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "play_audio(audio_rev_metas[0][\"audio_filepath\"])" ] }, { "cell_type": "code", "execution_count": 25, "id": "a21900ad", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "play_audio(audio_rev_metas[-1][\"audio_filepath\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "58a2ca6f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "41e4dbe5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "426ba2c3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dfc212cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "88a9e9cc", "metadata": {}, "source": [ "## Send stuff to rev" ] }, { "cell_type": "code", "execution_count": 26, "id": "05fbe672", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, upload_file, make_order, make_cc_order\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 27, "id": "2db71de1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "35.8 hours total\n" ] } ], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)\n", "\n", "ORDER_REF_STR = \"sl_gc_30h\"\n", "\n", "tot_duration_s = np.sum([m[\"duration_s\"] for m in audio_rev_metas])\n", "print(round(tot_duration_s / 60 / 60, 1), \"hours total\")" ] }, { "cell_type": "code", "execution_count": 28, "id": "8c66f7f9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [29:05<00:00, 36.37s/it]\n" ] } ], "source": [ "order_items = []\n", "for m in tqdm.tqdm(audio_rev_metas):\n", " media_loc = upload_file(AUTH_STR, m[\"audio_filepath\"], file_ref_str=m[\"uid\"])\n", " order_items.append(\n", " {\n", " \"media_loc\": media_loc,\n", " \"hotwords\": GLOBAL_HOTWORDS,\n", " \"duration_s\": int(np.ceil(m[\"duration_s\"])),\n", " \"uuid\": m[\"uid\"],\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": 29, "id": "a0ee5f4f", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"w\") as f:\n", " json.dump(order_items, f)" ] }, { "cell_type": "code", "execution_count": 30, "id": "8318e46f", "metadata": {}, "outputs": [], "source": [ "order_number = make_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)\n", "# order_number = make_cc_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)" ] }, { "cell_type": "code", "execution_count": 31, "id": "27549d51", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\"), \"w\") as f:\n", " f.write(order_number)" ] }, { "cell_type": "code", "execution_count": null, "id": "0096715b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "576b2bef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b33dc8e0", "metadata": {}, "source": [ "## Get rev stuff" ] }, { "cell_type": "code", "execution_count": 10, "id": "e8b59ece", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 11, "id": "9e4e40af", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\")) as f:\n", " order_number = f.read()" ] }, { "cell_type": "code", "execution_count": 12, "id": "d2a371b5", "metadata": {}, "outputs": [], "source": [ "# # grab while still in progress\n", "# from suno_utils.web.rev import _get_order_details, _parse_order_response\n", "# order_details = _get_order_details(AUTH_STR, order_number)\n", "# _, transcript_metas = _parse_order_response(order_details)" ] }, { "cell_type": "code", "execution_count": 58, "id": "9275ae0d", "metadata": {}, "outputs": [], "source": [ "# grab all of it\n", "transcript_metas = get_finished_order(AUTH_STR, order_number, blocking=False)" ] }, { "cell_type": "code", "execution_count": 59, "id": "df28d887", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [01:05<00:00, 1.37s/it]\n" ] } ], "source": [ "# 59 total\n", "for transcript_meta in tqdm.tqdm(transcript_metas):\n", " transcript_filepath = os.path.join(FROM_REV_DIR, transcript_meta[\"uuid\"] + \".json\")\n", " if os.path.exists(transcript_filepath):\n", " continue\n", " transcript_json = get_transcript(AUTH_STR, transcript_meta)\n", " with open(transcript_filepath, \"w\") as f:\n", " json.dump(transcript_json, f)" ] }, { "cell_type": "code", "execution_count": 60, "id": "ff8aab6d", "metadata": {}, "outputs": [], "source": [ "# !rm /mnt/data-ssd-1/data/private/customer/speechly/gymclass-30h/from_rev/452caf9f-dfc4-405b-8a00-9382e97b36a0.json" ] }, { "cell_type": "code", "execution_count": null, "id": "164bd5ca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "caaeacee", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b15ce9de", "metadata": {}, "source": [ "## Parsing transcripts" ] }, { "cell_type": "code", "execution_count": 71, "id": "00e04fcf", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 72, "id": "936b45dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "292/292 segments retained in 3f39e043-6f5b-4c5a-9fa8-8fa56683de6e.\n", "311/311 segments retained in 5f4ab58e-b34e-4745-9683-ccb7c9e3aea4.\n", "290/290 segments retained in 6ab8e1e0-7346-4eb8-b0fc-abcc98d7350e.\n", "296/296 segments retained in 9a6244d8-0bbc-4001-8713-c724151e7a58.\n", "323/323 segments retained in ebb3a70f-a861-4833-8926-0cafb23b8742.\n", "306/306 segments retained in 7438f36b-2b6a-4c23-b65b-13d4044d07df.\n", "315/315 segments retained in 571ef196-431e-4df2-b63d-86ccc12c5b94.\n", "312/312 segments retained in 98c03a64-6c18-4bc3-93a2-4b46e70afbc2.\n", "311/311 segments retained in dce9ff8a-f2fa-4b31-8358-1706270aa47f.\n", "311/311 segments retained in 213a2b60-3341-4f7f-a0a9-7524b31a7210.\n", "283/283 segments retained in eda53bcf-1ce8-4c86-97c7-62e7c0fa078b.\n", "310/310 segments retained in 0a1ba197-8fcd-42b9-b418-11b1fc6760f7.\n", "312/312 segments retained in a5026b1c-597a-4893-b2b2-6f6d545e529d.\n", "298/298 segments retained in e847d8f4-90d4-4300-805f-67a9bd259442.\n", "332/332 segments retained in 6c3d200c-dcf4-4fcb-87ae-7ce13796c2b6.\n", "327/327 segments retained in 2a5bf99a-0206-4ba7-b00a-5025d39200b8.\n", "311/311 segments retained in 368372ad-286d-40a5-96d8-3bec99754da1.\n", "301/301 segments retained in 87a65576-379b-40b3-9979-3259ede7a7bb.\n", "309/309 segments retained in cf7b8487-6dc6-4118-a473-2346fae776f0.\n", "301/301 segments retained in 006a7b7c-56b3-4948-a081-9333e6c401d6.\n", "302/302 segments retained in fb22088d-7c68-429f-8b61-ff938a07a8d0.\n", "305/305 segments retained in 671194bc-e484-4697-beb4-7bdafb01cb9e.\n", "299/299 segments retained in 8b6ab773-dd0c-4b83-8d68-ee455a86d955.\n", "325/325 segments retained in 21c92790-00ae-4693-bb58-02069c38e480.\n", "305/305 segments retained in e7a07630-7471-43dd-b6bb-37b6d33f206d.\n", "289/289 segments retained in eac08735-f77f-4697-bec3-4801d0a4633e.\n", "293/293 segments retained in 97ab380e-3ba4-491c-a369-f58bb448fd36.\n", "308/308 segments retained in 452caf9f-dfc4-405b-8a00-9382e97b36a0.\n", "271/271 segments retained in 36daf08f-4462-4140-a6eb-1401a9f859d8.\n", "324/324 segments retained in 9f3fa283-1908-467e-a915-0bd81ae91f6b.\n", "318/318 segments retained in 5beb19ff-4741-4905-b333-5e5147789ffb.\n", "304/304 segments retained in 3afa0868-e03e-4541-b058-a7d1f854537f.\n", "335/335 segments retained in 7f98e585-52cc-4a38-b416-5b00f8ca6269.\n", "302/302 segments retained in 8a1385ba-374c-4489-93d6-6bdee283b79d.\n", "302/302 segments retained in 3e82f2b9-8ba7-479c-b550-3d16d0cea44d.\n", "310/310 segments retained in 8980bdfb-6282-4283-9543-29a10958b945.\n", "308/308 segments retained in fda1be71-839e-4d96-bdbd-59656d869f5e.\n", "319/319 segments retained in c9cb6f47-664a-4e05-a7a1-dd33c04aa57e.\n", "288/288 segments retained in d794dba1-5cec-48c6-9687-9ed149d2234d.\n", "310/310 segments retained in 355e53e1-ef9d-4f81-ba81-21c2eeb19411.\n", "321/321 segments retained in 264b7eb8-00b6-4324-9cb1-400128bbc938.\n", "303/303 segments retained in d1510d2a-d253-4786-b01f-64477166556f.\n", "305/305 segments retained in 6b9dedbe-eae2-4e20-8882-ae5f8e7e43b1.\n", "300/300 segments retained in 27c7d26c-074d-4f66-a7d6-74ecd056a9a7.\n", "307/307 segments retained in 83c7c665-e049-4da2-9d91-1413a0689284.\n", "273/273 segments retained in bf3dd90f-7f31-416b-b7f2-c3d4d5f9b75a.\n", "319/319 segments retained in 7cc19755-0d32-4cd1-b71b-a9d7ac6320fa.\n", "177/177 segments retained in 431ab716-3fe5-4910-97e3-b4aa48a0dc66.\n" ] } ], "source": [ "from suno_utils.web.rev import parse_transcript\n", "from suno_utils.customers.sanas.pipeline import get_segments\n", "\n", "segment_transcripts = []\n", "for m in audio_rev_metas:#[3:4]:\n", " \n", "# if \"452caf9f-dfc4-40\" in m[\"uid\"]:\n", "# continue\n", " \n", " transcript_filepath = os.path.join(FROM_REV_DIR, m[\"uid\"] + \".json\")\n", " if not os.path.exists(transcript_filepath):\n", " print(\"not found, skipping...\")\n", " continue\n", " with open(transcript_filepath) as f:\n", " transcript_json = json.load(f)\n", " rev_transcript = parse_transcript(transcript_json)\n", " speech_segment_info = [mm for mm in m[\"segments_meta\"] if mm[\"type\"] == \"speech\"]\n", " n_segments = len(speech_segment_info)\n", " assert(len(speech_segment_info) == m[\"segments_meta\"][-1][\"segment_number\"] + 1)\n", " token_segments = get_segments(rev_transcript, n_segments, global_indicator_speaker=False, n_indicator_digits=3)\n", "\n", " segment_transcripts_chunk = []\n", " missing_ids = set(range(n_segments))\n", " for segment_idx, tokens in token_segments:\n", " segment_info = speech_segment_info[segment_idx]\n", " segment_transcripts_chunk.append({\n", " \"uid\": segment_info[\"orginial_audio_uid\"],\n", " \"duration_s\": segment_info[\"duration_s\"],\n", " \"transcript\": tokens.anonymize_speakers(),\n", " })\n", " missing_ids -= set([segment_idx])\n", " missing_ids = sorted([str(n) for n in missing_ids], key=lambda x: int(x))\n", " missing_str = \"\"\n", " if len(missing_ids) > 0:\n", " missing_str = \" (\" + \",\".join(missing_ids[:5])\n", " if len(missing_ids) > 5:\n", " missing_str += \"...\"\n", " missing_str += \")\"\n", " print(\"{}/{} segments retained in {}{}.\".format(\n", " len(segment_transcripts_chunk), n_segments, m[\"uid\"], missing_str\n", " ))\n", " segment_transcripts.extend(segment_transcripts_chunk)\n", " \n", "# break" ] }, { "cell_type": "code", "execution_count": 73, "id": "decb81d7", "metadata": {}, "outputs": [], "source": [ "# save as json\n", "import copy\n", "l = []\n", "for m in segment_transcripts:\n", " e = copy.deepcopy(m)\n", " e[\"transcript\"] = e[\"transcript\"].as_dict()\n", " l.append(e)\n", "with open(os.path.join(BASE_DIR, \"parsed_transcripts.json\"), \"w\") as f:\n", " json.dump(l, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "aeb27e5c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "68d01cb3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "52c54f78", "metadata": {}, "source": [ "## look at some basic stuff" ] }, { "cell_type": "code", "execution_count": 64, "id": "2cbe6494", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "037208d4-807a-412b-9d81-c1110ed71508\n", "You can go like this, one hand and I made it.\n", "----------\n", "0432176e-4b1f-4708-acff-004ad906064d\n", "Yeah. And you're hop -- high glitching, so I don't care.\n", "----------\n", "0474bf8a-3ef1-4f5f-b38f-d78e4706113f\n", "Wait, how did you get, wait, how, wait, when did you guys get two points? Wait, when did they get two points?\n", "----------\n", "0266aea1-e52c-460d-b16c-d734cf195f3e\n", "50, w -- 50 wins when he sen -- and these men can't dribble ?\n", "----------\n", "02e99de5-4adc-4b3f-9a32-ff782f1530bc\n", "\n", "----------\n" ] } ], "source": [ "for m in segment_transcripts[:5]:\n", " print(m[\"uid\"])\n", " print(m[\"transcript\"].text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 74, "id": "5a7a4e28", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "uid = \"0474bf8a-3ef1-4f5f-b38f-d78e4706113f\"\n", "Audio.from_file(os.path.join(AUDIO_DIR, uid + \".wav\")).play()" ] }, { "cell_type": "code", "execution_count": 546, "id": "136464c0", "metadata": {}, "outputs": [], "source": [ "# print(transcript_srt)" ] }, { "cell_type": "code", "execution_count": 543, "id": "50313c41", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "transcript id: 59511101-f9fe-4fd8-aff4-69927a0ad3e4\n" ] } ], "source": [ "segment_uid = \"3cb0cc5b-c5be-46a6-966b-2684c5fe4868\"\n", "for m in segments_manifest:\n", " if m[\"id\"] == segment_uid:\n", " print(\"transcript id:\", m[\"source_transcript_id\"])\n", " break" ] }, { "cell_type": "code", "execution_count": null, "id": "c2636a15", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c34c0db7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9937474e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2624f95e", "metadata": {}, "source": [ "## prepare pipeline" ] }, { "cell_type": "code", "execution_count": 74, "id": "9433c48a", "metadata": {}, "outputs": [], "source": [ "# map segment id to rev submission id\n", "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)\n", "segment_to_rev_id = {}\n", "for m in audio_rev_metas:\n", " rev_id = m[\"uid\"]\n", " for mm in m[\"segments_meta\"]:\n", " if mm[\"type\"] != \"speech\":\n", " continue\n", " segment_to_rev_id[mm[\"orginial_audio_uid\"]] = rev_id" ] }, { "cell_type": "code", "execution_count": 75, "id": "a29859c9", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"parsed_transcripts.json\")) as f:\n", " segment_transcripts = json.load(f)" ] }, { "cell_type": "code", "execution_count": 76, "id": "020e71bb", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 14573/14573 [00:00<00:00, 94199.33it/s]\n" ] } ], "source": [ "manifest = []\n", "for m in tqdm.tqdm(segment_transcripts):\n", " segment_filepath = os.path.join(AUDIO_DIR, m[\"uid\"] + \".wav\") \n", " new_segment_filepath = os.path.join(SEGMENTS_DIR, \"{}.wav\".format(m[\"uid\"]))\n", " if not os.path.exists(new_segment_filepath):\n", " audio = Audio.from_file(segment_filepath, sample_rate=16_000, byte_width=2)\n", " audio.to_wav(new_segment_filepath)\n", " manifest.append({\n", " \"id\": m[\"uid\"],\n", " \"uri\": new_segment_filepath,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"transcript\": {\"tokens\": m[\"transcript\"]},\n", " \"ignore_segment\": False,\n", " \"ignore_reason\": \"\",\n", " \"source_audio_offset_s\": 0,\n", " \"source_audio_uri\": segment_filepath,\n", " \"source_audio_id\": m[\"uid\"],\n", " \"source_transcript_id\": segment_to_rev_id[m[\"uid\"]],\n", " \"source_transcript_uri\": os.path.join(FROM_REV_DIR, segment_to_rev_id[m[\"uid\"]] + \".srt\"),\n", " })" ] }, { "cell_type": "code", "execution_count": 77, "id": "cee8873a", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\"), \"w\") as f:\n", " json.dump(manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "9aff6c62", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6cf2b71d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "651e980e", "metadata": {}, "source": [ "## QA" ] }, { "cell_type": "code", "execution_count": 78, "id": "a8dae9e1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "29.7 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 79, "id": "6a5f0e8c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "024914cc-6f85-4444-909e-b483900e635b\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "04102dbd-e06f-40a2-b1a4-3c7876a9ddc3\n", "No. I'm not playing guard, I'm not a point guard.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "033efde0-f58f-4d4f-b254-0990d33cef15\n", "Bro, what are you doing? Edgar, what are you doing? [laughter]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "036b168f-dd0d-46a2-861a-a5432132dffb\n", "And if, if -- Yeah, if you have stairs you can, uh, make yourself like taller than the room.\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "02ed7854-0a37-46bf-aca4-cf4e746ba04e\n", "Mad [laughter] he's mad. [inaudible]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = segments_manifest[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 489, "id": "081a8b29", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "45bfa253-011b-4027-9162-5d14f0403d58\n" ] } ], "source": [ "# get rev id for segment\n", "segment_uid = \"846622bb-b4ab-439c-9316-3683831bafbe\"\n", "for m in segments_manifest:\n", " if m[\"id\"] == segment_uid:\n", " print(m[\"source_transcript_id\"])\n", " break" ] }, { "cell_type": "code", "execution_count": null, "id": "a574e8b2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "88388672", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7b8ddb47", "metadata": {}, "source": [ "## run pipeline" ] }, { "cell_type": "code", "execution_count": 80, "id": "1e319351", "metadata": {}, "outputs": [], "source": [ "# # reset pipeline\n", "# !mv /mnt/data-ssd-1/data/private/customer/speechly/gymclass-30h/pipeline/2022_09_23/artifacts/01_segment_meta.json /tmp/01_segment_meta.json\n", "# !rm -rf /mnt/data-ssd-1/data/private/customer/speechly/gymclass-30h/pipeline/2022_09_23/artifacts/*\n", "# !mv /tmp/01_segment_meta.json /mnt/data-ssd-1/data/private/customer/speechly/gymclass-30h/pipeline/2022_09_23/artifacts/" ] }, { "cell_type": "code", "execution_count": 153, "id": "7fcf3139", "metadata": {}, "outputs": [], "source": [ "# # 17mins for 40h\n", "# CUDA_VISIBLE_DEVICES=0 python /home/georg/notebooks/customers/speechly/gymclass_30h/run_modified_pipeline.py" ] }, { "cell_type": "code", "execution_count": null, "id": "ec2da368", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fb1aec05", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "cfa1946d", "metadata": {}, "source": [ "## look at results" ] }, { "cell_type": "code", "execution_count": 100, "id": "30275498", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "29.7 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"02_segment_meta_post_norm_and_asr.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 101, "id": "de2eee4e", "metadata": {}, "outputs": [], "source": [ "l = []\n", "for e in segments_manifest:\n", " text = Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text\n", " text_norm = Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text\n", " if \"geez\" in text:\n", " l.append((e[\"id\"], text, text_norm))\n", "# print(e[\"id\"])\n", "# print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", "# print(Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n", "# Audio.from_file(e[\"uri\"]).play()\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 45, "id": "5848265d", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "03747378-3603-4e18-8122-cad57c40b876\n", "Uh. Uh. Uh. Yeah, mute me.\n", "uh uh uh yeah mute me\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "045bddd8-2690-4d43-b6a9-e8880fd5a88c\n", "It's a three point. It's all red. It's gonna take it back and my hands glitched now. Let's go, baby.\n", "it's a three point it's all red it's gonna take it back and my hands glitched now let's go baby\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "02108ff3-252a-40b2-837a-235c34d299c1\n", "Take my download. Wait. I got reasons. I'ma make all, love these mistakes.\n", "take my download wait i got reasons i ma make all love these mistakes\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "0382cfb9-c668-456a-9771-ce53dc65b809\n", "Dude, I jumped on purpose. That's so much already.\n", "dude i jumped on purpose that's so much already\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "042b44eb-6830-4a52-ba84-997e1bd82866\n", "No, I just need to hit a green, bro.\n", "no i just need to hit a green bro\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = segments_manifest[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 43, "id": "34165d5a", "metadata": {}, "outputs": [], "source": [ "# uid = \"6643b5a8-06de-45cb-a99f-351c54d49133\"\n", "# for e in segments_manifest:\n", "# if e[\"id\"] == uid:\n", "# print(\"full:\", Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", "# print(\"norm:\", Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n", "# print(\"asr :\", e[\"transcript_asr\"][\"transcript\"])\n", "# Audio.from_file(e[\"uri\"]).play()" ] }, { "cell_type": "code", "execution_count": 28, "id": "afd18433", "metadata": {}, "outputs": [], "source": [ "# l = []\n", "# for e in segments_manifest:\n", "# l.extend(re.findall(r\"[a-zA-Z]+\\-[a-zA-Z]+\", Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text))" ] }, { "cell_type": "code", "execution_count": 494, "id": "51e32035", "metadata": {}, "outputs": [], "source": [ "# # segment_uid = \"30815903-8a47-4d14-8557-41c352526d5d\"\n", "# # segment_uid = \"974c5846-5ae1-4521-9ca2-83175d563c55\"\n", "# # segment_uid = \"45a812db-16ef-494f-acde-6d151d012a70\"\n", "# segment_uid = \"383220cb-e8f1-4b6d-b39b-2d126eb50163\" \n", "\n", "# for n, m in enumerate(audio_rev_metas):\n", "# rev_id = m[\"uid\"]\n", "# for nn, mm in enumerate(m[\"segments_meta\"]):\n", "# if mm[\"type\"] != \"speech\":\n", "# continue\n", "# if mm[\"orginial_audio_uid\"] == segment_uid:\n", "# print(n, nn)\n", "# print(audio_rev_metas[n][\"uid\"])\n", "# print(audio_rev_metas[n][\"segments_meta\"][nn])\n", "# # print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)" ] }, { "cell_type": "code", "execution_count": null, "id": "539144e1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dc0d9e56", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 48, "id": "938c2793", "metadata": {}, "outputs": [], "source": [ "# !ls /mnt/data-ssd-1/data/private/customer/speechly/gymclass-30h/raw_data" ] }, { "cell_type": "markdown", "id": "6b0192cb", "metadata": {}, "source": [ "## format output for speechly styleguide" ] }, { "cell_type": "code", "execution_count": 200, "id": "7c3f2643", "metadata": {}, "outputs": [], "source": [ "# https://www.cs.cmu.edu/~biglou/resources/bad-words.txt\n", "with open(\"bad-words.txt\") as f:\n", " profane_words = set([s_clean for s in f.read().strip().split(\"\\n\") if len((s_clean := s.strip())) > 0])\n", "profane_words_ptn = \"|\".join(re.escape(s.lower()) for s in sorted(profane_words, key=len, reverse=True))\n", "PROFANE_WORDS_RE = re.compile(r\"\\b(?:\" + profane_words_ptn + r\")\\b\")\n", "\n", "def _contains_profanity(text):\n", " return bool(PROFANE_WORDS_RE.search(text.lower()))\n", "\n", "\n", "from nltk.corpus import words as nltk_words\n", "COMMON_EN_WORDS = set(nltk_words.words())\n", "\n", "def _partial_word_repl(m):\n", " s = m.group()\n", " word = s.strip(\"- \")\n", " probably_fragment = word.lower() not in COMMON_EN_WORDS\n", " if probably_fragment:\n", " return f\"{word}* -- \"\n", " else:\n", " return f\"{word} -- \"\n", " \n", "def _expand_casual(text):\n", " # 'cause -> cause, fittin', flippin' -> ing\n", " text = text.replace(\"'cause\", \"because\")\n", " text = re.sub(r\"in\\'(?=\\s|$)\", \"ing\", text)\n", " return text" ] }, { "cell_type": "code", "execution_count": 201, "id": "290b5f06", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.harvest import get_filename\n", "\n", "with open(os.path.join(BASE_DIR, \"raw_data_manifest.json\")) as f:\n", " filepaths_info = json.load(f)\n", "id_to_original_id_map = {e[0]: get_filename(e[2], keep_extension=True) for e in filepaths_info}\n", "inv_id_to_original_id_map = {v: k for k, v in id_to_original_id_map.items()}\n", "\n", "with open(os.path.join(ARTIFACTS_DIR, \"02_segment_meta_post_norm_and_asr.json\")) as f:\n", " segments_manifest = json.load(f)\n", " \n", "with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest_3 = json.load(f)" ] }, { "cell_type": "code", "execution_count": 243, "id": "0a0f096b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "498/14573 have multi speaker\n", "110 clips with background speakers found\n" ] } ], "source": [ "# get background speakers\n", "# 03cd712f-acd0-42d1-a77f-3114ed1d6814 - last no is background\n", "from collections import defaultdict\n", "multi_manifest = []\n", "for m in segments_manifest_3:\n", " speaker_ids = set([t.get(\"speaker_id\") for t in m[\"transcript\"][\"tokens\"] if t.get(\"speaker_id\") is not None])\n", " if len(speaker_ids) >= 2:\n", " multi_manifest.append(m)\n", "print(\"{}/{} have multi speaker\".format(len(multi_manifest), len(segments_manifest)))\n", "background_speakers = defaultdict(list)\n", "for m in multi_manifest:\n", " arr = Audio.from_file(m[\"uri\"]).array_float\n", " speaker_powers = defaultdict(list)\n", " for td in m[\"transcript_normalized\"][\"tokens\"]:\n", " if \"start_s\" not in td or \"speaker_id\" not in td or td[\"speaker_id\"] is None:\n", " continue\n", " arr_seg = arr[int(round(16_000*td[\"start_s\"])):int(round(16_000*td[\"end_s\"]))]\n", " speaker_powers[td[\"speaker_id\"]].append(np.mean(np.abs(arr_seg)))\n", " if len(speaker_powers) <= 1:\n", " continue\n", " speaker_power = {k: np.mean(v) for k, v in speaker_powers.items()}\n", " # background speakers\n", " max_pow = np.max(list(speaker_power.values()))\n", " for k, v in speaker_power.items():\n", " if v <= max_pow * 0.1:\n", " background_speakers[m[\"id\"]].append(k)\n", "# print(\"speaker {} is background\".format(k))\n", "print(len(background_speakers), \"clips with background speakers found\")" ] }, { "cell_type": "code", "execution_count": 258, "id": "ecd8a4d5", "metadata": {}, "outputs": [], "source": [ "def _convert_full_transcript(tokens, background_speaker_ids=None):\n", " transcript_l = []\n", " for turn in tokens.speaker_turns:\n", " turn_tokens = Tokens.from_dict(turn[\"tokens\"])\n", " active_lyrics = False\n", " text_l = []\n", " for t in turn_tokens:\n", " is_lyrics = False\n", " if t.metadata is not None:\n", " is_lyrics = t.metadata[\"is_lyrics\"]\n", " if is_lyrics and not active_lyrics:\n", " text_l.append(\"\")\n", " if not is_lyrics and active_lyrics:\n", " text_l.append(\"\")\n", " text_l.append(t.value)\n", " active_lyrics = is_lyrics\n", " if active_lyrics:\n", " text_l.append(\"\")\n", " text = \" \".join(text_l)\n", " text = re.sub(r\"\\\\s+\", \"\", text)\n", " transcript = re.sub(r\"\\s+\\<\\/lyrics\\>\", \"\", text)\n", " transcript = re.sub(r\"\\s+\\[word\\_fragment\\_boundary\\]\", \"*\", transcript)\n", " transcript = re.sub(r\"[A-Za-z][a-z]+ \\-\\-\", _partial_word_repl, transcript)\n", " transcript = transcript.replace(\"[foreign_language]\", \"\")\n", " transcript = transcript.replace(\"[inaudible]\", \"\")\n", " transcript = transcript.replace(\"[singing]\", \"\")\n", " transcript = transcript.replace(\"[crosstalk]\", \"\")\n", " transcript = transcript.replace(\"[redacted]\", \"\")\n", " transcript = transcript.replace(\"--\", \" \")\n", " transcript = re.sub(r\"\\[.*?\\]\", \" \", transcript)\n", " # fix some basic parse mistackes\n", " # transcript = re.sub(r\"^\\s*\\[music\\] \\[speaker\\_change\\]\\s*\", \"[music] \", transcript)\n", " # transcript = re.sub(r\"\\b([Ff])cuk\\b\", \"\\\\1uck\", transcript)\n", " transcript = _expand_casual(transcript)\n", " transcript = normalize_whitespace(transcript)\n", " if len(transcript) == 0:\n", " continue\n", " if background_speaker_ids is not None and turn[\"speaker_id\"] in background_speaker_ids:\n", " transcript = f\"[{transcript}]\"\n", " transcript_l.append(transcript)\n", "# transcript = normalize_whitespace(\" [speaker_change] \".join(transcript_l))\n", " transcript = normalize_whitespace(\" \".join(transcript_l))\n", " return transcript\n", "\n", "def _convert_norm_transcript(tokens, unnorm_text=None, background_speaker_ids=None):\n", " transcript_l = []\n", " for turn in tokens.speaker_turns:\n", " transcript = turn[\"text\"]\n", " transcript = transcript.replace(\" [word_fragment_boundary]\", \"*\")\n", " transcript = re.sub(r\"[A-Za-z][a-z]+ \\-\\-\", _partial_word_repl, transcript)\n", " transcript = transcript.replace(\"[foreign_language]\", \"\")\n", " transcript = transcript.replace(\"[inaudible]\", \"\")\n", " transcript = transcript.replace(\"[singing]\", \"\")\n", " transcript = transcript.replace(\"[crosstalk]\", \"\")\n", " transcript = transcript.replace(\"[redacted]\", \"\")\n", " transcript = transcript.replace(\"--\", \" \")\n", " transcript = re.sub(r\"\\[.*?\\]\", \" \", transcript)\n", " transcript = _expand_casual(transcript)\n", " # teanscript = re.sub(r\"\\bfcuk\\b\", \"fuck\", transcript)\n", " # add back in dashes\n", " if unnorm_text is not None:\n", " for word in re.findall(r\"[a-z]+(?:\\-[a-z]+)+\", unnorm_text.lower()):\n", " transcript = transcript.replace(word.replace(\"-\", \" \"), word)\n", " transcript = normalize_whitespace(transcript)\n", " if len(transcript) == 0:\n", " continue\n", " if background_speaker_ids is not None and turn[\"speaker_id\"] in background_speaker_ids:\n", " transcript = f\"[{transcript}]\"\n", " transcript_l.append(transcript)\n", " transcript = normalize_whitespace(\" \".join(transcript_l))\n", " return transcript" ] }, { "cell_type": "code", "execution_count": 259, "id": "b2465b35", "metadata": {}, "outputs": [], "source": [ "# jsonl \n", "\n", "# {\n", "# \"audio\": \"0a4ac9ac-42a4-4e77-91ad-862b0509d913.wav.\", // the audio file name\n", "# \"transcript\": \".....\", // The transcript of the audio\n", "# \"tags\": [ ] // The \"OFF\" and/or \"AC\" tags if relevant to the utterance\n", "# }\n", "# Audio corrupted utterance (AC): for only severely corrupted audio files\n", "# Offensive utterance (OFF): profanities, bullying language, hate speech included in the audio file\n", "output_metadata = []\n", "for n, m in enumerate(segments_manifest):\n", " norm_tokens = Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"])\n", " tokens = Tokens.from_dict(m[\"transcript\"][\"tokens\"])\n", " norm_transcript = _convert_norm_transcript(\n", " norm_tokens, unnorm_text=tokens.text, background_speaker_ids=background_speakers[m[\"id\"]]\n", " )\n", " transcript = _convert_full_transcript(tokens, background_speaker_ids=background_speakers[m[\"id\"]])\n", " tags = []\n", " if _contains_profanity(norm_transcript):\n", " tags.append(\"OFF\")\n", " original_filename = id_to_original_id_map[m[\"id\"]]\n", " \n", "# if \"foreign\" in transcript:\n", "# print(norm_transcript)\n", "# print(\"-\")\n", "# print(transcript)\n", "# print(\"-\"*10)\n", "# if len(norm_transcript) == 0:\n", "# print(norm_transcript)\n", "# print(\"-\")\n", "# print(transcript)\n", "# print(\"-\"*10)\n", " \n", " # some manual overrides\n", " # TODO: use CER here in future to punt and maybe reannotate\n", "# if original_filename == \"c7bfc752-be04-4c91-b018-bf809ad19323.mp3\":\n", "# transcript = transcript.replace(\"Banja not to this barely\", \"[foreign_language] Chef Bally\")\n", "# norm_transcript = norm_transcript.replace(\"we ' re\", \"we're\")\n", "# if original_filename == \"cc6284ac-dc8b-420c-9bd0-8470d2e2d88d.mp3\":\n", "# transcript = \"Damn.\"\n", "# norm_transcript = \"damn\"\n", "# tags = [\"OFF\"]\n", "# if original_filename == \"b451b2ff-0dd2-4a8b-a317-34ea67f172bd.mp3\":\n", "# transcript = transcript.replace(\"I can't\", \"I can't, I can't\")\n", "# norm_transcript = norm_transcript.replace(\"i can't\", \"i can't i can't\")\n", "# if original_filename == \"4600e16e-fd7d-4c69-9e1d-49b1a3c2b088.mp3\":\n", "# transcript = transcript.replace(\"y'all\", \"yo\")\n", "# norm_transcript = norm_transcript.replace(\"y'all\", \"yo\") \n", "# if original_filename == \"3cd8a90f-8475-4d7c-84f3-07e9b7aa0227.mp3\":\n", "# transcript = \"What?\"\n", "# norm_transcript = \"what\"\n", "# if original_filename == \"558a3f70-8f55-4b7e-9b14-6d6e2a436218.mp3\":\n", "# transcript = \"\"\n", "# norm_transcript = \"[inaudible]\"\n", "# if original_filename == \"35063d27-f049-41d1-a157-417ea07f068c.mp3\":\n", "# transcript = transcript.replace(\"one zero\", \"ten\")\n", "\n", " output_metadata.append({\n", " \"audio\": original_filename,\n", " \"transcript\": normalize_whitespace(norm_transcript),\n", " \"transcript_full\": normalize_whitespace(transcript),\n", " \"tags\": tags,\n", " })" ] }, { "cell_type": "code", "execution_count": 260, "id": "00cecd18", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"delivery_meta.jsonl\"), \"w\") as f:\n", " for e in output_metadata:\n", " f.write(json.dumps(e))\n", " f.write(\"\\n\")" ] }, { "cell_type": "code", "execution_count": 261, "id": "ba591032", "metadata": {}, "outputs": [], "source": [ "# !cat /mnt/data-ssd-1/data/private/customer/speechly/gymclass-30h/pipeline/2022_09_23/artifacts/delivery_meta.jsonl | grep 03cd712f-acd0-42d1-a77f-3114ed1d6814" ] }, { "cell_type": "code", "execution_count": null, "id": "96e9eeb8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "79b64c80", "metadata": {}, "source": [ "## investigate background speaker" ] }, { "cell_type": "code", "execution_count": 262, "id": "ed893539", "metadata": {}, "outputs": [], "source": [ "# 03cd712f-acd0-42d1-a77f-3114ed1d6814 - last no is background\n", "\n", "# for m in multi_manifest[:20]:\n", "# if m[\"id\"] == \"03cd712f-acd0-42d1-a77f-3114ed1d6814\":\n", "# print(\"found\")\n", "# break" ] }, { "cell_type": "code", "execution_count": 238, "id": "14a91199", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 498/498 [00:00<00:00, 2840.36it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "110 clips with background speakers found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# get background speakers\n", "# 03cd712f-acd0-42d1-a77f-3114ed1d6814 - last no is background\n", "from collections import defaultdict\n", "multi_manifest = []\n", "for m in segments_manifest_3:\n", " speaker_ids = set([t.get(\"speaker_id\") for t in m[\"transcript\"][\"tokens\"] if t.get(\"speaker_id\") is not None])\n", " if len(speaker_ids) >= 2:\n", " multi_manifest.append(m)\n", "print(\"{}/{} have multi speaker\".format(len(multi_manifest), len(segments_manifest)))\n", "background_speakers = defaultdict(list)\n", "for m in tqdm.tqdm(multi_manifest):\n", " arr = Audio.from_file(m[\"uri\"]).array_float\n", " speaker_powers = defaultdict(list)\n", " for td in m[\"transcript_normalized\"][\"tokens\"]:\n", " if \"start_s\" not in td or \"speaker_id\" not in td or td[\"speaker_id\"] is None:\n", " continue\n", " arr_seg = arr[int(round(16_000*td[\"start_s\"])):int(round(16_000*td[\"end_s\"]))]\n", " speaker_powers[td[\"speaker_id\"]].append(np.mean(np.abs(arr_seg)))\n", " if len(speaker_powers) <= 1:\n", " continue\n", " speaker_power = {k: np.mean(v) for k, v in speaker_powers.items()}\n", " # background speakers\n", " max_pow = np.max(list(speaker_power.values()))\n", " for k, v in speaker_power.items():\n", " if v <= max_pow * 0.1:\n", " background_speakers[m[\"id\"]].append(k)\n", "# print(\"speaker {} is background\".format(k))\n", "print(len(background_speakers), \"clips with background speakers found\")" ] }, { "cell_type": "code", "execution_count": 207, "id": "b6c2c361", "metadata": { "scrolled": false }, "outputs": [], "source": [ "# for m in multi_manifest[:20]:\n", "# Audio.from_file(m[\"uri\"]).play()\n", "# print(m[\"id\"])\n", "# print(Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 246, "id": "489accab", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "03cd712f-acd0-42d1-a77f-3114ed1d6814\n", "All right. Y'all ready? Y'all ready? You ready? No.\n", "----------\n" ] } ], "source": [ "n = 0\n", "for m in segments_manifest:\n", " if m[\"id\"] not in background_speakers:\n", " continue\n", " if m[\"id\"] != \"03cd712f-acd0-42d1-a77f-3114ed1d6814\":\n", " continue\n", " Audio.from_file(m[\"uri\"]).play()\n", " print(m[\"id\"])\n", " tokens = Tokens.from_dict(m[\"transcript\"][\"tokens\"])\n", " background_speaker_ids = background_speakers[m[\"id\"]]\n", " print(tokens.text)\n", " print(\"-\"*10)\n", " n += 1\n", " if n == 20:\n", " break\n", " break" ] }, { "cell_type": "code", "execution_count": 250, "id": "7a3baa6d", "metadata": {}, "outputs": [], "source": [ "text_turns = []\n", "for st in tokens.speaker_turns:\n", " text_turns.append((st[\"speaker_id\"], st[\"text\"]))" ] }, { "cell_type": "code", "execution_count": 248, "id": "8f586d27", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[1]" ] }, "execution_count": 248, "metadata": {}, "output_type": "execute_result" } ], "source": [ "background_speaker_ids" ] }, { "cell_type": "code", "execution_count": null, "id": "73cddfb8", "metadata": {}, "outputs": [], "source": [ "tokens = Tokens.from_dict(m[\"transcript\"][\"tokens\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "378c87be", "metadata": {}, "outputs": [], "source": [ "tokens, background_speaker_ids=None" ] }, { "cell_type": "code", "execution_count": null, "id": "facad161", "metadata": {}, "outputs": [], "source": [ " tokens = Tokens.from_dict(m[\"transcript\"][\"tokens\"])\n", " norm_transcript = _convert_norm_transcript(\n", " norm_tokens, unnorm_text=tokens.text, background_speaker_ids=background_speakers[m[\"id\"]]\n", " )\n", " transcript = _convert_full_transcript(tokens, background_speaker_ids=background_speakers[m[\"id\"]])" ] }, { "cell_type": "code", "execution_count": null, "id": "87fb0f60", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e0f0dcf3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "628d31ef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6b0b614f", "metadata": {}, "source": [ "## Final QA" ] }, { "cell_type": "code", "execution_count": 263, "id": "a61aa689", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"delivery_meta.jsonl\")) as f:\n", " output_metadata = [json.loads(e) for e in f.read().strip().split(\"\\n\")]" ] }, { "cell_type": "code", "execution_count": 171, "id": "8a5d9f26", "metadata": {}, "outputs": [], "source": [ "# for e in output_metadata:\n", "# if e[\"audio\"] == \"1034ef38-79aa-48de-a908-0dd6a65c1a36.mp3\":\n", "# print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", "# Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", "# print(e[\"transcript\"])\n", "# print(\"-\")\n", "# print(e[\"transcript_full\"])\n", "# break" ] }, { "cell_type": "code", "execution_count": 172, "id": "201ac267", "metadata": {}, "outputs": [], "source": [ "# # figure out rev\n", "# with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", "# audio_rev_metas = json.load(f)\n", "# uid = inv_id_to_original_id_map[\"harrisnt_-7072462733219728682.m4v\"]\n", "# for rev_m in audio_rev_metas:\n", "# for seg_m in rev_m[\"segments_meta\"]:\n", "# if seg_m.get(\"orginial_audio_uid\") == uid:\n", "# print(seg_m)" ] }, { "cell_type": "code", "execution_count": 173, "id": "9ccfd86b", "metadata": {}, "outputs": [], "source": [ "# import random\n", "# # random ones\n", "# # l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "# l = [e for e in output_metadata if len(e[\"transcript\"]) == 0]\n", "# random.shuffle(l)\n", "# for e in l[:5]:\n", "# print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", "# Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", "# print(e[\"transcript\"])\n", "# print(\"-\")\n", "# print(e[\"transcript_full\"])\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 264, "id": "a3f6a7dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "03b1a6b1-2d39-4b4f-b990-051b39267e89.wav - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "-\n", "\n", "----------\n", "042ee43e-1482-4670-b509-6fc35b715b19.wav - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "are live but yet they don't talk to you ugly a oop\n", "-\n", "Are live but yet they don't talk to you ugly a oop.\n", "----------\n", "037e7e16-bed9-4a2d-aeba-ec428717ee72.wav - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "you turtle hey hey hey that's oh that's a miss hey you're trash\n", "-\n", "You turtle, hey, hey, hey, that's oh, that's a miss. Hey. You're trash.\n", "----------\n", "02b2c67a-38da-4cdd-b334-1e1600fb6df1.wav - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "almost\n", "-\n", "Almost\n", "----------\n", "04215da1-7241-4b46-ad5c-603374dc217a.wav - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "like emo trick shot\n", "-\n", "Like, Emo trick shot.\n", "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = output_metadata[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", " Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", " print(e[\"transcript\"])\n", " print(\"-\")\n", " print(e[\"transcript_full\"])\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "76c34c77", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a33601a9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "9f84953b", "metadata": {}, "source": [ "## compare with their output" ] }, { "cell_type": "code", "execution_count": 35, "id": "08368ccc", "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 33, "id": "18112b59", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "11105 transcripts\n" ] }, { "data": { "text/plain": [ "[('09e56ba8-49c6-4301-802a-13a9d25d56aa', 'ohh fuck bro had a bounce'),\n", " ('081b33a7-128f-46f3-a40b-e83114bc8752',\n", " \"bro i snatch blocked him bro what the fuck no god this game is so delayed with that stuff it's not hate socially\")]" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "with open(os.path.join(RAW_DATA_DIR, \"transcripts.tsv\")) as f:\n", " sl_transcripts = [e.split(\"\\t\") for e in f.read().strip().split(\"\\n\")]\n", "assert(all([len(e) == 2 for e in sl_transcripts]))\n", "sl_transcripts = [(uid, text.lower()) for uid, text in sl_transcripts]\n", "print(len(sl_transcripts), \"transcripts\")\n", "sl_transcripts[:2]" ] }, { "cell_type": "code", "execution_count": 36, "id": "e038f68f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-09-29 17:35:38 nemo_logging:349] /tmp/ipykernel_3407955/2405280702.py:1: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only\n", " test_df = pd.DataFrame(output_metadata).drop([\"tags\"], 1)\n", " \n" ] } ], "source": [ "test_df = pd.DataFrame(output_metadata).drop([\"tags\"], 1)\n", "test_df[\"uid\"] = test_df[\"audio\"].str.split(\".\").str[0]\n", "test_df[\"sl_transcript\"] = test_df[\"uid\"].map({k: v for k, v in sl_transcripts})" ] }, { "cell_type": "code", "execution_count": 37, "id": "b1f33880", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.metrics import get_wer\n", "test_df[\"wer\"] = [get_wer(a, b) for a, b in zip(test_df[\"transcript\"], test_df[\"sl_transcript\"])]\n", "test_df = test_df.sort_values(\"wer\", ascending=False).reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 38, "id": "a66aa1b0", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
audiotranscripttranscript_fulluidsl_transcriptwer
9ddf85902-0e28-4e22-9959-dbd2fe335efc.wavdingo <u> mute mute hey my friend made a fanma...Dingo! [inaudible] Mute, mute. Hey my friend m...ddf85902-0e28-4e22-9959-dbd2fe335efcbingo new new new new very good opening pingu ...1.0
278630b1cd-b4ce-47d6-b283-9346d3737e71.wavdam what the hell <u> bitch you're a concernin...Dam, what the hell? [crosstalk] [speaker_chang...8630b1cd-b4ce-47d6-b283-9346d3737e711.0
29d0b09d5e-71d8-4c76-848e-831ac750da10.wavbitch shut the fuck up hold up shut the fuck u...[music] Bitch. Shut the fuck up. Hold up. Shut...d0b09d5e-71d8-4c76-848e-831ac750da101.0
30977c3b1d-0304-4f91-b96b-cfb0497c5ead.wavi'm the best player couldn't get rid of him if...<lyrics>I'm the best player</lyrics> [speaker_...977c3b1d-0304-4f91-b96b-cfb0497c5eadfucking you represent it's for you you can't c...1.0
33aaa597f0-4946-444f-a7a3-eec6bf4b26f7.wav<u> eat dirt eat dirt eat dirt eat dirt me tim...[crosstalk] [speaker_change] Eat dirt, eat dir...aaa597f0-4946-444f-a7a3-eec6bf4b26f7better retard he died retard1.0
\n", "
" ], "text/plain": [ " audio \\\n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc.wav \n", "27 8630b1cd-b4ce-47d6-b283-9346d3737e71.wav \n", "29 d0b09d5e-71d8-4c76-848e-831ac750da10.wav \n", "30 977c3b1d-0304-4f91-b96b-cfb0497c5ead.wav \n", "33 aaa597f0-4946-444f-a7a3-eec6bf4b26f7.wav \n", "\n", " transcript \\\n", "9 dingo mute mute hey my friend made a fanma... \n", "27 dam what the hell bitch you're a concernin... \n", "29 bitch shut the fuck up hold up shut the fuck u... \n", "30 i'm the best player couldn't get rid of him if... \n", "33 eat dirt eat dirt eat dirt eat dirt me tim... \n", "\n", " transcript_full \\\n", "9 Dingo! [inaudible] Mute, mute. Hey my friend m... \n", "27 Dam, what the hell? [crosstalk] [speaker_chang... \n", "29 [music] Bitch. Shut the fuck up. Hold up. Shut... \n", "30 I'm the best player [speaker_... \n", "33 [crosstalk] [speaker_change] Eat dirt, eat dir... \n", "\n", " uid \\\n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc \n", "27 8630b1cd-b4ce-47d6-b283-9346d3737e71 \n", "29 d0b09d5e-71d8-4c76-848e-831ac750da10 \n", "30 977c3b1d-0304-4f91-b96b-cfb0497c5ead \n", "33 aaa597f0-4946-444f-a7a3-eec6bf4b26f7 \n", "\n", " sl_transcript wer \n", "9 bingo new new new new very good opening pingu ... 1.0 \n", "27 1.0 \n", "29 1.0 \n", "30 fucking you represent it's for you you can't c... 1.0 \n", "33 better retard he died retard 1.0 " ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_df[test_df[\"transcript\"].str.len() >= 50].head()" ] }, { "cell_type": "code", "execution_count": 39, "id": "dc008f15", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
audiotranscripttranscript_fulluidsl_transcriptwer
1798a79cc-96df-409b-82ee-143bf66940c9.wav<u> bro my shoot upon oh shut up like <u>[inaudible] [speaker_change] Bro my shoot upon...798a79cc-96df-409b-82ee-143bf66940c9then leave are you fucking dumb are you really...1.0
575df8c55-72cf-4c2e-a0f3-54d1cd714787.wavgimme dat oh fuck oh my gosh <u>Gimme dat. Oh fuck, oh my gosh. [inaudible]75df8c55-72cf-4c2e-a0f3-54d1cd714787i'm gonna be that ohf for my boy my mom is1.0
68f260fcb-f33a-4d99-aac0-c2d79acbd644.wav<u> watch y'all when i'm fucked[crosstalk] [speaker_change] Watch y'all when ...8f260fcb-f33a-4d99-aac0-c2d79acbd6441.0
9ddf85902-0e28-4e22-9959-dbd2fe335efc.wavdingo <u> mute mute hey my friend made a fanma...Dingo! [inaudible] Mute, mute. Hey my friend m...ddf85902-0e28-4e22-9959-dbd2fe335efcbingo new new new new very good opening pingu ...1.0
113d9c372e-526f-4803-8bee-3362469c0b7f.wav<u>[inaudible]3d9c372e-526f-4803-8bee-3362469c0b7fwell fucking1.0
\n", "
" ], "text/plain": [ " audio \\\n", "1 798a79cc-96df-409b-82ee-143bf66940c9.wav \n", "5 75df8c55-72cf-4c2e-a0f3-54d1cd714787.wav \n", "6 8f260fcb-f33a-4d99-aac0-c2d79acbd644.wav \n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc.wav \n", "11 3d9c372e-526f-4803-8bee-3362469c0b7f.wav \n", "\n", " transcript \\\n", "1 bro my shoot upon oh shut up like \n", "5 gimme dat oh fuck oh my gosh \n", "6 watch y'all when i'm fucked \n", "9 dingo mute mute hey my friend made a fanma... \n", "11 \n", "\n", " transcript_full \\\n", "1 [inaudible] [speaker_change] Bro my shoot upon... \n", "5 Gimme dat. Oh fuck, oh my gosh. [inaudible] \n", "6 [crosstalk] [speaker_change] Watch y'all when ... \n", "9 Dingo! [inaudible] Mute, mute. Hey my friend m... \n", "11 [inaudible] \n", "\n", " uid \\\n", "1 798a79cc-96df-409b-82ee-143bf66940c9 \n", "5 75df8c55-72cf-4c2e-a0f3-54d1cd714787 \n", "6 8f260fcb-f33a-4d99-aac0-c2d79acbd644 \n", "9 ddf85902-0e28-4e22-9959-dbd2fe335efc \n", "11 3d9c372e-526f-4803-8bee-3362469c0b7f \n", "\n", " sl_transcript wer \n", "1 then leave are you fucking dumb are you really... 1.0 \n", "5 i'm gonna be that ohf for my boy my mom is 1.0 \n", "6 1.0 \n", "9 bingo new new new new very good opening pingu ... 1.0 \n", "11 well fucking 1.0 " ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_df[test_df[\"transcript_full\"].str.contains(r\"\\[\")].head()" ] }, { "cell_type": "code", "execution_count": 40, "id": "faca5ede", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "4b235196-f1e3-4184-932a-18ed4315eb07\n", "bye nigga bye nigga\n", "buying buy\n" ] } ], "source": [ "n = 23\n", "row = test_df.iloc[n]\n", "Audio.from_file(os.path.join(AUDIO_DIR, row[\"audio\"])).play()\n", "print(row[\"uid\"])\n", "print(row[\"transcript\"])\n", "print(row[\"sl_transcript\"])" ] }, { "cell_type": "code", "execution_count": 415, "id": "22240080", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "15 642\n", "2f60e11b-7fbb-498a-86e4-efaeb67a0f17\n", "{'segment_number': 160, 'type': 'speech', 'duration_s': 34.24, 'orginial_audio_offset_s': 0, 'orginial_audio_uid': '6643b5a8-06de-45cb-a99f-351c54d49133'}\n" ] } ], "source": [ "# segment_uid = \"30815903-8a47-4d14-8557-41c352526d5d\"\n", "# segment_uid = \"974c5846-5ae1-4521-9ca2-83175d563c55\"\n", "# segment_uid = \"45a812db-16ef-494f-acde-6d151d012a70\"\n", "segment_uid = \"6643b5a8-06de-45cb-a99f-351c54d49133\" \n", "\n", "for n, m in enumerate(audio_rev_metas):\n", " rev_id = m[\"uid\"]\n", " for nn, mm in enumerate(m[\"segments_meta\"]):\n", " if mm[\"type\"] != \"speech\":\n", " continue\n", " if mm[\"orginial_audio_uid\"] == segment_uid:\n", " print(n, nn)\n", " print(audio_rev_metas[n][\"uid\"])\n", " print(audio_rev_metas[n][\"segments_meta\"][nn])\n", "# print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)" ] }, { "cell_type": "code", "execution_count": null, "id": "fc2a4bce", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7fc6b380", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bc513d9e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d51f20f8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "36ae3093", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "d3d86de5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a6f077fb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d394bb5b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ff7c5d47", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }