{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "b304dfff", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "f955e852", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "e9716000", "metadata": {}, "outputs": [], "source": [ "import time\n", "import os\n", "import tqdm\n", "import shutil\n", "import random\n", "import funcy\n", "import json\n", "import numpy as np\n", "import multiprocessing\n", "import pandas as pd\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.conversion import play_audio, get_audio_properties\n", "from suno_utils.utils.slicer import get_vad_gaps, get_intervals_from_gaps\n", "\n", "\n", "DATE_DIR = \"2022_08_07\"\n", "\n", "BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/\"\n", "TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\", DATE_DIR)\n", "FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\", DATE_DIR)\n", "TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n", "REV_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"rev_manifest.jsonl\")\n", "FAILED_MANIFEST_FILEPATH = os.path.join(TO_REV_DIR, \"failed_manifest.jsonl\")\n", "\n", "# SEGMENT_DIR = os.path.join(BASE_DIR, \"pipeline/2022_08_07/artifacts/segments\")" ] }, { "cell_type": "code", "execution_count": 32, "id": "2d612fd7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "29.1h of segments, 32.4h of rev submission\n" ] } ], "source": [ "with open(REV_MANIFEST_FILEPATH) as f:\n", " rev_submission_meta = []\n", " for p in f.read().strip().split(\"\\n\"):\n", " rev_submission_meta.append(json.loads(p))\n", "seg_tot_s = 0\n", "tot_s = 0\n", "for m in rev_submission_meta:\n", " seg_tot_s += np.sum([s[\"duration_s\"] for s in m[\"segments_meta\"] if s[\"type\"] == \"speech\"])\n", " tot_s += np.sum([s[\"duration_s\"] for s in m[\"segments_meta\"]])\n", "seg_tot_s = round(seg_tot_s / 60 / 60, 1)\n", "tot_s = round(tot_s / 60 / 60, 1)\n", "print(\"{}h of segments, {}h of rev submission\".format(seg_tot_s, tot_s))" ] }, { "cell_type": "code", "execution_count": null, "id": "3c451ead", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9a054d3d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c9c31c96", "metadata": {}, "source": [ "## Parse" ] }, { "cell_type": "code", "execution_count": 6, "id": "3a4802e1", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import parse_transcript\n", "\n", "rev_transcripts = []\n", "for fn in os.listdir(FROM_REV_DIR):\n", " if not fn.endswith(\".json\"):\n", " continue\n", " with open(os.path.join(FROM_REV_DIR, fn)) as f:\n", " raw_transcript = json.load(f)\n", " transcript = parse_transcript(raw_transcript, anonymize_speakers=True)\n", " rev_transcripts.append({\n", " \"id\": fn[:-5],\n", " \"transcript\": transcript,\n", " })" ] }, { "cell_type": "code", "execution_count": 59, "id": "514e7fe3", "metadata": {}, "outputs": [], "source": [ "# find most egregious mistakes\n", "from collections import defaultdict\n", "\n", "duration_info = defaultdict(dict)\n", "\n", "for m in rev_submission_meta:\n", " duration_info[m[\"uid\"]][\"available_duration_s\"] = np.sum([\n", " s[\"duration_s\"] for s in m[\"segments_meta\"] if s[\"type\"] == \"speech\"\n", " ])\n", " \n", "for m in segments_meta:\n", " uid = m[\"source_audio_id\"]\n", " if \"retained_duration_s\" not in duration_info[uid]:\n", " duration_info[uid][\"retained_duration_s\"] = 0\n", " duration_info[uid][\"retained_duration_s\"] += m[\"duration_s\"]\n", " \n", "df = pd.DataFrame(duration_info).T.rename_axis(\"uid\").fillna(0)\n", "df[\"missing_frac\"] = 1 - df[\"retained_duration_s\"] / df[\"available_duration_s\"]\n", "df[\"missing_duration_s\"] = df[\"available_duration_s\"] - df[\"retained_duration_s\"]" ] }, { "cell_type": "code", "execution_count": 60, "id": "4ec46a66", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
available_duration_sretained_duration_smissing_fracmissing_duration_s
uid
f26fa631-f017-402a-8e7c-9da308e910bf860.550.01.0860.55
03a2e7d8-8133-4b73-a7c0-bba239aa5f0d647.080.01.0647.08
a2c28cfd-287e-4c08-93c8-7600e54789f1607.170.01.0607.17
7b636c96-6e40-41be-b944-5dbe24a50b27598.350.01.0598.35
31206718-1d70-451c-80dd-b4c3907a6982579.940.01.0579.94
\n", "
" ], "text/plain": [ " available_duration_s \\\n", "uid \n", "f26fa631-f017-402a-8e7c-9da308e910bf 860.55 \n", "03a2e7d8-8133-4b73-a7c0-bba239aa5f0d 647.08 \n", "a2c28cfd-287e-4c08-93c8-7600e54789f1 607.17 \n", "7b636c96-6e40-41be-b944-5dbe24a50b27 598.35 \n", "31206718-1d70-451c-80dd-b4c3907a6982 579.94 \n", "\n", " retained_duration_s missing_frac \\\n", "uid \n", "f26fa631-f017-402a-8e7c-9da308e910bf 0.0 1.0 \n", "03a2e7d8-8133-4b73-a7c0-bba239aa5f0d 0.0 1.0 \n", "a2c28cfd-287e-4c08-93c8-7600e54789f1 0.0 1.0 \n", "7b636c96-6e40-41be-b944-5dbe24a50b27 0.0 1.0 \n", "31206718-1d70-451c-80dd-b4c3907a6982 0.0 1.0 \n", "\n", " missing_duration_s \n", "uid \n", "f26fa631-f017-402a-8e7c-9da308e910bf 860.55 \n", "03a2e7d8-8133-4b73-a7c0-bba239aa5f0d 647.08 \n", "a2c28cfd-287e-4c08-93c8-7600e54789f1 607.17 \n", "7b636c96-6e40-41be-b944-5dbe24a50b27 598.35 \n", "31206718-1d70-451c-80dd-b4c3907a6982 579.94 " ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.sort_values(\"missing_duration_s\", ascending=False).head()" ] }, { "cell_type": "code", "execution_count": 310, "id": "90a7f695", "metadata": {}, "outputs": [], "source": [ "from statistics import mode\n", "\n", "digits = list(\"0123456789\")\n", "numbers = \"zero one two three four five six seven eight nine\".split()\n", "\n", "def _is_numeric(text):\n", " text_bits = re.sub(r\"[^\\sa-z0-9]\", \"\", text.lower()).strip().split()\n", " return all([s in set(numbers) or re.match(r\"[0-9]+\", s) for s in text_bits])\n", "\n", "def _digitify(text):\n", " for d, n in zip(digits, numbers):\n", " text = text.lower().replace(n, d)\n", " text = re.sub(r\"[^0-9]\", \"\", text)\n", " return text\n", "\n", "def _find_subs(s1, s2):\n", " subs = []\n", " loc = 0\n", " while s1:\n", " s1_copy = s1\n", " while s1_copy:\n", " while s1_copy and s1_copy not in s2:\n", " s1_copy = s1_copy[:-1]\n", " if s1_copy:\n", " subs.append((loc, s2.index(s1_copy), s1_copy))\n", " loc += len(s1_copy)\n", " s1 = s1[len(s1_copy):]\n", " else:\n", " s1 = s1[1:]\n", " loc += 1\n", " s1_copy = s1 \n", " return subs\n", "\n", "def get_segments(rev_transcript, n_segments):\n", " \n", " # find indicator speaker\n", " indicator_segments = [\n", " t[\"speaker_id\"] for t in rev_transcript.speaker_turns if _is_numeric(t[\"plaintext\"])\n", " ]\n", " if len(indicator_segments) == 0:\n", " print(\"no indicators found\")\n", " return []\n", " speaker_id = mode(indicator_segments)\n", " # parse numbers for indicator speaker\n", " indicator_tokens = []\n", " prev_n = -2\n", " for n, token in enumerate(rev_transcript.tokens):\n", " if token.speaker_id != speaker_id:\n", " continue\n", " if n - prev_n == 1:\n", " indicator_tokens[-1] = (indicator_tokens[-1][0], indicator_tokens[-1][1] + _digitify(token.value))\n", " else:\n", " indicator_tokens.append((n, _digitify(token.value)))\n", " prev_n = n\n", "\n", " # see which of the indicator tokens got matched\n", " # list of [(indicator_index, token_index)]\n", "\n", " match_str = \"*\".join([s for _, s in indicator_tokens])\n", " known_idx_2_segment_nr = {}\n", " indicator_texts = []\n", " offs_idx = 0\n", " for n in range(n_segments):\n", " text = str(n).zfill(2)\n", " for nn in range(len(text)):\n", " known_idx_2_segment_nr[offs_idx + nn] = n\n", " offs_idx += len(text) + 1\n", " indicator_texts.append(text)\n", " known_str = \"|\".join(indicator_texts)\n", " matched_idx_set = {}\n", "# from difflib import SequenceMatcher\n", "# matcher = SequenceMatcher(\n", "# a=match_str,\n", "# b=known_str,\n", "# autojunk=False,\n", "# )\n", "# for block in matcher.get_matching_blocks():\n", "# if block.size == 0:\n", "# continue\n", "# l = []\n", "# for n in range(block.b, block.b + block.size):\n", "# l.append(known_idx_2_segment_nr[n])\n", "# if len(set(l)) != 1:\n", "# continue\n", "# matched_idx_set[block.a] = l[0]\n", " \n", " for a_idx, b_idx, match_g in _find_subs(match_str, known_str):\n", " l = []\n", " for n in range(b_idx, b_idx + len(match_g)):\n", " l.append(known_idx_2_segment_nr[n])\n", " if len(set(l)) != 1:\n", " continue\n", " matched_idx_set[a_idx] = l[0]\n", " \n", " \n", " offs_idx = 0\n", " correct_indicators = [] # indicator_index, token_index\n", " for token_idx, text in indicator_tokens:\n", " for n in range(len(text)):\n", " if offs_idx + n in matched_idx_set:\n", " correct_indicators.append((matched_idx_set[offs_idx + n], token_idx)) \n", " offs_idx += len(text) + 1\n", "\n", " # get relevant tokens for all consecutive ones\n", " token_segments = []\n", " for (indicator_index_l, token_index_l), (indicator_index_r, token_index_r) in zip(\n", " correct_indicators[:-1], correct_indicators[1:]\n", " ):\n", " if indicator_index_l == indicator_index_r - 1:\n", " relevant_tokens = [\n", " t for t in rev_transcript.tokens[token_index_l:token_index_r] if t.speaker_id != speaker_id\n", " ]\n", " token_segments.append((indicator_index_l, relevant_tokens))\n", " # check last one\n", " indicator_index_l, token_index_l = correct_indicators[-1]\n", " if indicator_index_l == n_segments - 1:\n", " relevant_tokens = [\n", " t for t in rev_transcript.tokens[token_index_l:] if t.speaker_id != speaker_id\n", " ]\n", " token_segments.append((indicator_index_l, relevant_tokens))\n", " \n", " return token_segments" ] }, { "cell_type": "code", "execution_count": 183, "id": "6ae43490", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "14.1 minutes\n" ] } ], "source": [ "for m in rev_transcripts:\n", " if m[\"id\"] == \"f26fa631-f017-402a-8e7c-9da308e910bf\":\n", " rev_transcript = m[\"transcript\"]\n", " break\n", "for m in rev_submission_meta:\n", " if m[\"uid\"] == \"f26fa631-f017-402a-8e7c-9da308e910bf\":\n", " segments_meta = m[\"segments_meta\"]\n", " break\n", " \n", "n_segments = segments_meta[-1][\"segment_number\"] + 1\n", "token_segments = get_segments(rev_transcript, n_segments)\n", "segments_retained = set([n for n, _ in token_segments])\n", "dur_s = 0\n", "for m in segments_meta:\n", " if m[\"type\"] == \"speech\" and m[\"segment_number\"] in segments_retained:\n", " dur_s += m[\"duration_s\"]\n", "print(round(dur_s / 60, 1), \"minutes\")" ] }, { "cell_type": "code", "execution_count": 316, "id": "809fac82", "metadata": {}, "outputs": [], "source": [ "# segments_meta_map = {}\n", "# for m in rev_submission_meta:\n", "# segments_meta_map[m[\"uid\"]] = m[\"segments_meta\"]\n", "# durs_list = []\n", "# for m in tqdm.tqdm(rev_transcripts):\n", "# segments_meta = segments_meta_map[m[\"id\"]]\n", "# rev_transcript = m[\"transcript\"]\n", "# n_segments = segments_meta[-1][\"segment_number\"] + 1\n", "# token_segments = get_segments(rev_transcript, n_segments)\n", "# segments_retained = set([n for n, _ in token_segments])\n", "# dur_s = 0\n", "# for m in segments_meta:\n", "# if m[\"type\"] == \"speech\" and m[\"segment_number\"] in segments_retained:\n", "# dur_s += m[\"duration_s\"]\n", "# durs_list.append(dur_s)\n", "# print(round(np.sum(durs_list) / 60 / 60, 1), \"hours\")\n", "# # 28.8 hours" ] }, { "cell_type": "code", "execution_count": 340, "id": "ca3521fe", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 9%|█████████████▏ | 60/698 [00:26<04:15, 2.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "no indicators found\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", " 9%|█████████████▎ | 61/698 [00:27<06:08, 1.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "low segment coverage for f431333a-2ed2-4d00-89ad-542886d96646, skipping...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 698/698 [05:22<00:00, 2.17it/s]\n" ] } ], "source": [ "# write the manifest\n", "import uuid\n", "\n", "segments_meta_map = {}\n", "for m in rev_submission_meta:\n", " segments_meta_map[m[\"uid\"]] = m\n", "manifest = []\n", "for m in tqdm.tqdm(rev_transcripts):\n", " segments_meta = segments_meta_map[m[\"id\"]][\"segments_meta\"]\n", " rev_transcript = m[\"transcript\"]\n", " n_segments = segments_meta[-1][\"segment_number\"] + 1\n", " token_segments = get_segments(rev_transcript, n_segments)\n", " meta_info_map = {}\n", " for mm in segments_meta:\n", " if mm[\"type\"] != \"speech\":\n", " continue\n", " meta_info_map[mm[\"segment_number\"]] = mm\n", " source_audio_uri = segments_meta_map[m[\"id\"]][\"original_audio_filepath\"]\n", " audio = Audio.from_file(source_audio_uri, sample_rate=16_000, byte_width=2)\n", " if len(token_segments) / n_segments < 0.6:\n", " print(\"low segment coverage for {}, skipping...\".format(m[\"id\"]))\n", " continue\n", " for n, tokens in token_segments:\n", " segment_id = str(uuid.uuid4())\n", " new_segment_uri = os.path.join(BASE_DIR, \"pipeline/2022_08_07__2/segments/{}.wav\".format(segment_id))\n", " from_s = meta_info_map[n][\"orginial_audio_offset_s\"]\n", " to_s = from_s + meta_info_map[n][\"duration_s\"]\n", " audio.get_slice(from_s, to_s).to_wav(new_segment_uri)\n", " manifest.append({\n", " \"source_audio_offset_s\": from_s,\n", " \"id\": segment_id,\n", " \"uri\": new_segment_uri,\n", " \"duration_s\": meta_info_map[n][\"duration_s\"],\n", " \"transcript\": {\"tokens\": Tokens(tokens).as_dict()},\n", " \"ignore_segment\": False,\n", " \"ignore_reason\": \"\",\n", " \"source_audio_uri\": source_audio_uri,\n", " \"source_audio_id\": m[\"id\"],\n", " \"source_transcript_id\": m[\"id\"],\n", " \"source_transcript_uri\": os.path.join(BASE_DIR, \"from_rev/2022_08_07/{}.json\".format(m[\"id\"])),\n", " })" ] }, { "cell_type": "code", "execution_count": 339, "id": "1a19d6b7", "metadata": {}, "outputs": [], "source": [ "# !rm -rf /mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/pipeline/2022_08_07__2/segments\n", "# !mkdir /mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/pipeline/2022_08_07__2/segments" ] }, { "cell_type": "code", "execution_count": 341, "id": "cee8873a", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"pipeline/2022_08_07__2/artifacts/01_segment_meta.json\"), \"w\") as f:\n", " json.dump(manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "7e68ecbd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "95f1129b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8af21712", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dc5e491f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "acce0257", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "02c45969", "metadata": {}, "source": [ "## QA" ] }, { "cell_type": "code", "execution_count": null, "id": "c020ff15", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Tokens" ] }, { "cell_type": "code", "execution_count": 425, "id": "e1333bc9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "28.8 hours\n" ] } ], "source": [ "# with open(os.path.join(BASE_DIR, \"pipeline/2022_08_07/artifacts/01_segment_meta.json\")) as f:\n", "# with open(os.path.join(BASE_DIR, \"pipeline/2022_08_07__2/artifacts/01_segment_meta.json\")) as f:\n", "with open(os.path.join(BASE_DIR, \"pipeline/2022_08_07__2/artifacts/03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 416, "id": "aa498657", "metadata": {}, "outputs": [], "source": [ "# # random ones\n", "# # l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "# l = segments_manifest[:]\n", "# random.shuffle(l)\n", "# for e in l[:10]:\n", "# print(e[\"id\"])\n", "# print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", "# Audio.from_file(e[\"uri\"]).play()\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 428, "id": "8809e14d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'continuum service desk my name is c j can i have your winid'" ] }, "execution_count": 428, "metadata": {}, "output_type": "execute_result" } ], "source": [ "segments_manifest[0][\"transcript_normalized\"][\"text\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "70c564e1", "metadata": {}, "outputs": [], "source": [ "# sort by cer\n", "from suno_utils.utils.metrics import get_cer\n", "\n", "l = segments_manifest[:]\n", "cer_list = []\n", "for m in l:\n", " m[\"cer\"] = get_cer(m[\"transcript_normalized\"][\"text\"], m[\"transcript_asr\"][\"transcript\"])\n", "l = sorted(l, key=lambda x: x[\"cer\"], reverse=True)" ] }, { "cell_type": "code", "execution_count": 434, "id": "f22ef807", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'source_audio_offset_s': 92.41,\n", " 'id': 'd7eb9ea8-1c6e-4f2f-b1b5-4a768e2f929e',\n", " 'uri': '/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/pipeline/2022_08_07__2/segments/d7eb9ea8-1c6e-4f2f-b1b5-4a768e2f929e.wav',\n", " 'duration_s': 4.2,\n", " 'transcript': {'tokens': [], 'text': ''},\n", " 'ignore_segment': True,\n", " 'ignore_reason': 'high asr cer',\n", " 'source_audio_uri': '/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/Continuum-PH-ASR-1/GusDaniel_Gozar_Male_PH-Tagalog_1658931573072.wav',\n", " 'source_audio_id': '08a4ffd0-2f93-4375-9bdb-a35799e47847',\n", " 'source_transcript_id': '08a4ffd0-2f93-4375-9bdb-a35799e47847',\n", " 'source_transcript_uri': '/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/from_rev/2022_08_07/08a4ffd0-2f93-4375-9bdb-a35799e47847.json',\n", " 'transcript_normalized': {'tokens': [], 'text': ''},\n", " 'transcript_asr': {'model_id': 'd752f336-4815-4db1-be85-05ccce9000b4__EncDecCTCModelBPE',\n", " 'transcript': 'i never after you after you nplug the power cable press and hold'},\n", " 'cer': 1.0}" ] }, "execution_count": 434, "metadata": {}, "output_type": "execute_result" } ], "source": [ "l[3]" ] }, { "cell_type": "code", "execution_count": 432, "id": "a9841d3b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1.0\n", "6870edb1-c8a1-4754-9f2f-b9767d869117\n", "[Foreign language]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "0899f681-c56b-412d-8732-ec0a13c05531\n", "[Foreign language]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "7acdf50d-579c-4018-bd0b-ca67f91a0a93\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "d7eb9ea8-1c6e-4f2f-b1b5-4a768e2f929e\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "0de46388-71b9-478e-a151-4efc4961b812\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "709199d5-26f3-47c3-a720-e98d0b072dee\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "c8a7b2ee-68ba-403b-b569-de8ff2b73b35\n", "Uh, [inaudible]\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "1196fe2c-1ab9-4f50-9b9c-d5966eedcdcf\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "fd556fff-5a6d-4ad5-98d1-f4a78e0b4940\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "1.0\n", "1dbe6bbb-a6c0-4e1d-b92b-dfa0a144907e\n", "\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "for e in l[:10]:\n", " print(e[\"cer\"])\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 423, "id": "2eb3fffa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "28.8 hours\n" ] } ], "source": [ "with open(os.path.join(BASE_DIR, \"pipeline/2022_08_07__2/artifacts/04_segment_metadata.jsonl\")) as f:\n", " segments_manifest = [json.loads(e) for e in f.read().strip().split(\"\\n\")]\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 424, "id": "10a71342", "metadata": {}, "outputs": [], "source": [ "# d = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/to_sanas/2022-08-12/\"\n", "\n", "# # l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "# l = segments_manifest[:]\n", "# random.shuffle(l)\n", "# for e in l[:10]:\n", "# print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", "# Audio.from_file(d + e[\"uri\"]).play()\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "650a11cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bccfbf76", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "0f3f2a83", "metadata": {}, "source": [ "## run whole pipeline" ] }, { "cell_type": "code", "execution_count": null, "id": "bdd8803d", "metadata": {}, "outputs": [], "source": [ "# python run_modified_pipeline.py" ] }, { "cell_type": "code", "execution_count": 357, "id": "17efc28a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.1 hours\n" ] } ], "source": [ "with open(os.path.join(BASE_DIR, \"pipeline/2022_08_07__2/artifacts/03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)\n", " \n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")\n", "# 23.2 hours" ] }, { "cell_type": "code", "execution_count": null, "id": "2c30e932", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2b0e5be9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "1747220c", "metadata": {}, "source": [ "## find dominant speaker" ] }, { "cell_type": "code", "execution_count": 399, "id": "a393ebe6", "metadata": {}, "outputs": [], "source": [ "with open(REV_MANIFEST_FILEPATH) as f:\n", " rev_submission_meta = []\n", " for p in f.read().strip().split(\"\\n\"):\n", " rev_submission_meta.append(json.loads(p))\n", "original_id_map = {}\n", "for e in rev_submission_meta:\n", " original_id_map[e[\"uid\"]] = \"/\".join(e[\"original_audio_filepath\"].split(\"/\")[-2:])" ] }, { "cell_type": "code", "execution_count": 401, "id": "fdc45b00", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 698/698 [00:04<00:00, 153.93it/s]\n" ] } ], "source": [ "from suno_utils.web.rev import parse_transcript\n", "from collections import Counter\n", "\n", "main_speaker_info = []\n", "for fn in tqdm.tqdm(os.listdir(FROM_REV_DIR)):\n", " if not fn.endswith(\".json\"):\n", " continue\n", " uid = fn[:-5]\n", " original_uid = original_id_map[uid]\n", " with open(os.path.join(FROM_REV_DIR, fn)) as f:\n", " raw_transcript = json.load(f)\n", " transcript = parse_transcript(raw_transcript, anonymize_speakers=True)\n", " speech_amount = Counter()\n", " for t in transcript.tokens:\n", " if t.type != \"text\" or _is_numeric(t.value):\n", " continue\n", " speech_amount[t.speaker_id] += len(t.value)\n", " if len(speech_amount) == 0:\n", " main_speaker_info.append({\n", " \"original_filename\": original_uid,\n", " \"main_speakers\": [],\n", " })\n", " continue\n", " denom = np.sum([e[-1] for e in speech_amount.most_common()])\n", " main_speakers = [e[0] for e in speech_amount.most_common() if e[1] / denom >= 0.2]\n", "# if uid == \"b592becf-4d7b-4df1-afa0-1502be991d76\":\n", "# break\n", " main_speaker_info.append({\n", " \"original_filename\": original_uid,\n", " \"main_speakers\": main_speakers,\n", " })" ] }, { "cell_type": "code", "execution_count": 403, "id": "bd8741ab", "metadata": {}, "outputs": [], "source": [ "with open(\"main_speaker_info.json\", \"w\") as f:\n", " json.dump(main_speaker_info, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "aacae1d6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a7b1ffa7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "737a06b4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1d1075d4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "69ccc2cc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "024bbeb8", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 333, "id": "b66e48ad", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 8%|████████████▊ | 59/698 [00:00<00:03, 164.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "15327bfd-8b24-4b67-9b5f-8173d51669f3 0.7931034482758621\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "TEST_UUID = \"15327bfd-8b24-4b67-9b5f-8173d51669f3\"\n", "# TEST_UUID = \"15f15e13-62af-4eed-9f62-043ce54d95e0\"\n", "# TEST_UUID = \"ab8a3845-b192-44ae-b19f-03052ae2d49f\"\n", "\n", "segments_meta_map = {}\n", "for m in rev_submission_meta:\n", " segments_meta_map[m[\"uid\"]] = m\n", "manifest = []\n", "for m in tqdm.tqdm(rev_transcripts):\n", " if m[\"id\"] != TEST_UUID:\n", " continue\n", " segments_meta = segments_meta_map[m[\"id\"]][\"segments_meta\"]\n", " rev_transcript = m[\"transcript\"]\n", " n_segments = segments_meta[-1][\"segment_number\"] + 1\n", " token_segments = get_segments(rev_transcript, n_segments)\n", " meta_info_map = {}\n", " for mm in segments_meta:\n", " if mm[\"type\"] != \"speech\":\n", " continue\n", " meta_info_map[mm[\"segment_number\"]] = mm\n", " source_audio_uri = segments_meta_map[m[\"id\"]][\"original_audio_filepath\"]\n", " audio = Audio.from_file(source_audio_uri, sample_rate=16_000, byte_width=2)\n", " if len(token_segments) / n_segments < 0.999:\n", " print(m[\"id\"], len(token_segments) / n_segments)\n", " break" ] }, { "cell_type": "code", "execution_count": 334, "id": "0f772448", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.7931034482758621\n" ] } ], "source": [ "# find indicator speaker\n", "indicator_segments = [\n", " t[\"speaker_id\"] for t in rev_transcript.speaker_turns if _is_numeric(t[\"plaintext\"])\n", "]\n", "speaker_id = mode(indicator_segments)\n", "# parse numbers for indicator speaker\n", "indicator_tokens = []\n", "prev_n = -2\n", "for n, token in enumerate(rev_transcript.tokens):\n", " if token.speaker_id != speaker_id:\n", " continue\n", " if n - prev_n == 1:\n", " indicator_tokens[-1] = (indicator_tokens[-1][0], indicator_tokens[-1][1] + _digitify(token.value))\n", " else:\n", " indicator_tokens.append((n, _digitify(token.value)))\n", " prev_n = n\n", "\n", "# see which of the indicator tokens got matched\n", "# list of [(indicator_index, token_index)]\n", "from difflib import SequenceMatcher\n", "match_str = \"*\".join([s for _, s in indicator_tokens])\n", "known_idx_2_segment_nr = {}\n", "indicator_texts = []\n", "offs_idx = 0\n", "for n in range(n_segments):\n", " text = str(n).zfill(2)\n", " for nn in range(len(text)):\n", " known_idx_2_segment_nr[offs_idx + nn] = n\n", " offs_idx += len(text) + 1\n", " indicator_texts.append(text)\n", "known_str = \"|\".join(indicator_texts)\n", "matched_idx_set = {}\n", "for a_idx, b_idx, match_g in _find_subs(match_str, known_str):\n", " l = []\n", " for n in range(b_idx, b_idx + len(match_g)):\n", " l.append(known_idx_2_segment_nr[n])\n", " if len(set(l)) != 1:\n", " continue\n", " matched_idx_set[a_idx] = l[0]\n", " \n", "offs_idx = 0\n", "correct_indicators = [] # indicator_index, token_index\n", "for token_idx, text in indicator_tokens:\n", " for n in range(len(text)):\n", " if offs_idx + n in matched_idx_set:\n", " correct_indicators.append((matched_idx_set[offs_idx + n], token_idx)) \n", " offs_idx += len(text) + 1\n", "\n", "# get relevant tokens for all consecutive ones\n", "token_segments = []\n", "for (indicator_index_l, token_index_l), (indicator_index_r, token_index_r) in zip(\n", " correct_indicators[:-1], correct_indicators[1:]\n", "):\n", " if indicator_index_l == indicator_index_r - 1:\n", " relevant_tokens = [\n", " t for t in rev_transcript.tokens[token_index_l:token_index_r] if t.speaker_id != speaker_id\n", " ]\n", " token_segments.append((indicator_index_l, relevant_tokens))\n", "# check last one\n", "indicator_index_l, token_index_l = correct_indicators[-1]\n", "if indicator_index_l == n_segments - 1:\n", " relevant_tokens = [\n", " t for t in rev_transcript.tokens[token_index_l:] if t.speaker_id != speaker_id\n", " ]\n", " token_segments.append((indicator_index_l, relevant_tokens))\n", " \n", "print(len(token_segments) / n_segments)" ] }, { "cell_type": "code", "execution_count": null, "id": "b40e745c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e7bd0c79", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f021dc22", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "daf36b9a", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }