{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "435e772c",
"metadata": {},
"outputs": [],
"source": [
"# INFO:__main__:copying manifest file /mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest.json to /mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/00_segment_manifest.json\n",
"# INFO:__main__:Step `0_segmentation` starting...\n",
"# WARNING:suno_utils.datasets.rev_podcast.process_data:only 33.7% segment coverage for `/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/rev_files/3045f69a-52ca-4c35-8e83-7f0df9894889.json`.\n",
"# WARNING:suno_utils.datasets.rev_podcast.process_data:skipping `/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/rev_files/3045f69a-52ca-4c35-8e83-7f0df9894889.json` due to low coverage and likely mistakes.\n",
"# WARNING:suno_utils.datasets.rev_podcast.process_data:skipping `1f10a7e4-86b9-4df8-a445-a487e6044c06` due to segmenting error.\n",
"# INFO:suno_utils.datasets.rev_podcast.process_data:84.1% segment coverage after slicing\n",
"# INFO:suno_utils.datasets.rev_podcast.pipeline:8063 total segments\n",
"# INFO:__main__:Step `0_segmentation` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/01_segment_meta.json`.\n",
"# INFO:__main__:Step `0_segmentation` done. Took 9748s, total time: 9748s\n",
"# INFO:__main__:Step `1_norm+asr` starting...\n",
"# INFO:suno_utils.datasets.rev_podcast.process_data:transcribing 8063 segments using ASR\n",
"# Transcribing: 100%|██████████████████████████████████████████████████████| 1008/1008 [03:55<00:00, 4.28it/s]\n",
"# INFO:__main__:Step `1_norm+asr` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/02_segment_meta_post_norm_and_asr.json`.\n",
"# INFO:__main__:Step `1_norm+asr` done. Took 20444s, total time: 30192s\n",
"# INFO:__main__:Step `2_timestamps` starting...\n",
"# WARNING:suno_utils.web.mfa:12/8063 failed.\n",
"# INFO:suno_utils.datasets.rev_podcast.process_data:12/8063 failed for alignment.\n",
"# WARNING:suno_utils.web.mfa:10/8063 failed.\n",
"# INFO:suno_utils.datasets.rev_podcast.process_data:10/8063 failed for normalized alignment\n",
"# INFO:__main__:Step `2_timestamps` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/03_segment_meta_post_timestamp.json`.\n",
"# INFO:__main__:Step `2_timestamps` done. Took 11435s, total time: 41627s\n",
"# INFO:__main__:Step `3_distribution` starting...\n",
"# INFO:suno_utils.datasets.rev_podcast.pipeline:ignoring 212 of 8063 segments\n",
"# INFO:__main__:Step `3_distribution` wrote to `/mnt/data-ssd-1/data/gdata-mate/pipeline/artifacts/04_segment_meta_distribution.jsonl`.\n",
"# INFO:__main__:Step `3_distribution` done. Took 66s, total time: 41693s\n",
"# INFO:suno_utils.datasets.rev_podcast.pipeline:81.9% final segment coverage"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "537da831",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "34aa4a2a",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "cd9a17fe",
"metadata": {},
"outputs": [],
"source": [
"import logging\n",
"logging.basicConfig(level=logging.DEBUG)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "4b2667eb",
"metadata": {},
"outputs": [],
"source": [
"DATA_DIR = \"/mnt/data-ssd-1/data/gdata-mate/\"\n",
"RAW_DATA_DIR = os.path.join(DATA_DIR, \"pre_pipeline/rev_files/\")\n",
"\n",
"transcript_filepath = os.path.join(RAW_DATA_DIR, \"3045f69a-52ca-4c35-8e83-7f0df9894889.json\")\n",
"audio_filepath = os.path.join(RAW_DATA_DIR, \"3045f69a-52ca-4c35-8e83-7f0df9894889.wav\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "59871d27",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:suno_utils.tasks.ctc_align:8.8% WER\n",
"INFO:suno_utils.tasks.ctc_align:92.5% alignment fraction\n"
]
}
],
"source": [
"import json\n",
"from suno_utils.web.rev import parse_transcript\n",
"from suno_utils.tasks.ctc_align import align_transcript\n",
"\n",
"with open(transcript_filepath) as f:\n",
" transcript = json.load(f)\n",
"transcript_tokens = parse_transcript(transcript)\n",
"kenlm_bin_dir = \"/home/georg/tools/kenlm/build/bin\"\n",
"timestamped_transcript_tokens = align_transcript(audio_filepath, transcript_tokens, kenlm_bin_dir=kenlm_bin_dir)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "282af9b1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:suno_utils.utils.slicer_v2:94.2% coverage\n",
"INFO:suno_utils.utils.slicer_v2:74.3% coverage (without excluded)\n"
]
}
],
"source": [
"from suno_utils.audio.conversion import get_audio_properties\n",
"from suno_utils.utils.slicer_v2 import get_segments\n",
"\n",
"tot_durations_s = get_audio_properties(audio_filepath)[\"duration_s\"]\n",
"segments = get_segments(\n",
" timestamped_transcript_tokens, tot_durations_s,\n",
" min_silence_gap_s=0.2,\n",
" min_confidence=0.4,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ac92c440",
"metadata": {},
"outputs": [],
"source": [
"94.2% coverage\n",
"74.3% coverage (without excluded)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4f34a66e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f8922261",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "72c05cfc",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "e967573e",
"metadata": {},
"source": [
"#### check slices"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7bdeabf9",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.audio import Audio\n",
"import random\n",
"\n",
"random.seed(6006)\n",
"audio = Audio.from_file(audio_filepath)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "f8c8461f",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"(354.23, 362.01)\n",
"Um, Jim, before I let you go, [inaudible] the Aussie Open starts Monday week. It's gonna be huge. Uh, who do you think are the chances for the Aussies, um, in terms of,\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"(278.31, 289.13)\n",
"uh, Hazelwood and Starc have plateaued a bit. Uh, Cummins is, you know, I think a superstar. But, what happens if Nathan Lyon goes over? There's not a lot of other spinners knocking down the door. So, um,\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"(241.51, 258.92)\n",
"Well, you've actually, uh, answered my question. Um, I was gonna say, \"Is this Indian attack probably one of the best attacks in the world at the moment?\" Uh, you'd have to say yes, [inaudible] Yeah. Shami -- Shami and Bumrah, you know, they -- they -- they're a good, you know, Jadeja. I mean, they -- and, Ashwan hasn't even been there, and Sharma's out injured, [inaudible] Sharma. So, I mean, it's -- they're a good -- they're a good attack.\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"(348.64, 354.23)\n",
"You [inaudible] a bit worried about that one in the World Cup is All England's certainly one of the best teams in the world and probably deserve to be favorites in that.\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"(416.14, 432.67)\n",
"yeah, I think it's, um, it's gonna be interesting. As far as the Aussies go, I mean, I think Ash Barty had a wonderful 2018. Alex deMinaur had a very good, uh, very good 2018. Whether they can go well into the second week is a big question mark for me. Um, and -- and -- and Nick Kyrgios. Well, what are you gonna get with Nick? I mean, you -- you just -- hello.\n",
"----------\n"
]
}
],
"source": [
"# check segments\n",
"sample_segments = segments[:]\n",
"random.shuffle(sample_segments)\n",
"for (start_s, end_s), tokens in sample_segments[:5]:\n",
" audio.get_slice(start_s, end_s).play()\n",
" print(f\"({round(start_s, 2)}, {round(end_s, 2)})\")\n",
" print(tokens.text)\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "54ab73ba",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d912c91f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d3051e88",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf9fe9db",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "e4cef61b",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "a34dd59d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "ef09081a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "ff797390",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "da6a8605",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "b2041515",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "a072cc1e",
"metadata": {},
"outputs": [],
"source": [
"DATA_DIR = \"/mnt/data-ssd-1/data/gdata-mate/\"\n",
"RAW_DATA_DIR = os.path.join(DATA_DIR, \"pre_pipeline/rev_files/\")\n",
"# GENTLE_DATA_DIR = os.path.join(DATA_DIR, \"pipeline/alignment-gentle/\")\n",
"\n",
"transcript_filepath = os.path.join(RAW_DATA_DIR, \"3045f69a-52ca-4c35-8e83-7f0df9894889.json\")\n",
"audio_filepath = os.path.join(RAW_DATA_DIR, \"3045f69a-52ca-4c35-8e83-7f0df9894889.wav\")\n",
"# gentle_filepath = os.path.join(GENTLE_DATA_DIR, \"3045f69a-52ca-4c35-8e83-7f0df9894889.json\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "6378ecec",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"from suno_utils.web.rev import parse_transcript\n",
"from suno_utils.utils.text import normalize_whitespace\n",
"from suno_utils.audio import Token, Tokens\n",
"\n",
"def _normalize_token(text):\n",
" # remove all quotes other than inside word: eg. I'm, we're\n",
" text = text.lower()\n",
" text = re.sub(r\"^[^a-z0-9]+\", \"\", text)\n",
" text = re.sub(r\"[^a-z0-9]+$\", \"\", text)\n",
" text = normalize_whitespace(text)\n",
" return text\n",
"\n",
"def normalize_tokens(transcript_tokens):\n",
" # TODO: proper normalize while keeping track of token indices (can do with approx align)\n",
" tokens_normalized = []\n",
" for idx, token in enumerate(transcript_tokens):\n",
" if token.type != \"text\":\n",
" continue\n",
" clean_token = _normalize_token(token.value)\n",
" if len(clean_token) == 0:\n",
" continue\n",
" tokens_normalized.append(Token(\n",
" clean_token,\n",
" \"text\",\n",
" speaker_id=token.speaker_id,\n",
" metadata={\"idx_orig\": idx}\n",
" ))\n",
" return Tokens(tokens_normalized)\n",
"\n",
"\n",
"with open(transcript_filepath) as f:\n",
" transcript = json.load(f)\n",
"transcript_tokens = parse_transcript(transcript)\n",
"transcript_tokens_norm = normalize_tokens(transcript_tokens)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "3738a850",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Token(value='well', type='text', speaker_id='Bevo', start_s=None, end_s=None, metadata={'idx_orig': 0}),\n",
" Token(value='good', type='text', speaker_id='Bevo', start_s=None, end_s=None, metadata={'idx_orig': 1})]"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"transcript_tokens_norm[:2]"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "055f10eb",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"8.9% WER\n"
]
}
],
"source": [
"from suno_utils.tasks.asr import LmConfig, transcribe\n",
"from suno_utils.audio.conversion import get_audio_properties\n",
"from suno_utils.utils.metrics import get_wer\n",
"from IPython.utils.io import capture_output\n",
"import sys\n",
"import subprocess\n",
"import tempfile\n",
"\n",
"\n",
"def _write_kenlm_model(kenlm_bin_dir, train_text, out_model_filepath, ngrams=5):\n",
" with tempfile.TemporaryDirectory() as tmp_dir:\n",
" # write train text\n",
" train_text_filepath = os.path.join(tmp_dir, \"text.txt\")\n",
" with open(train_text_filepath, \"w\") as f:\n",
" f.write(train_text)\n",
"\n",
" # train LM\n",
" ngrams = 5\n",
" model_arpa_filepath = os.path.join(tmp_dir, \"model.arpa\")\n",
" train_kenlm_args = [\n",
" os.path.join(kenlm_bin_dir, \"lmplz\"),\n",
" \"-o\",\n",
" str(ngrams),\n",
" \"--text\",\n",
" train_text_filepath,\n",
" \"--arpa\",\n",
" model_arpa_filepath,\n",
" \"--discount_fallback\",\n",
" ]\n",
" out = subprocess.run(\n",
" train_kenlm_args, capture_output=False, text=True, stdout=sys.stdout, stderr=sys.stderr,\n",
" )\n",
" if out.returncode != 0:\n",
" raise RuntimeError(\"Training KenLM was not successful!\")\n",
"\n",
" # build binary\n",
" model_bin_filepath = os.path.join(tmp_dir, \"model.bin\")\n",
" kenlm_args = [\n",
" os.path.join(kenlm_bin_dir, \"build_binary\"),\n",
" \"-s\",\n",
" \"trie\",\n",
" model_arpa_filepath,\n",
" out_model_filepath,\n",
" ]\n",
" out = subprocess.run(\n",
" kenlm_args, capture_output=False, text=True, stdout=sys.stdout, stderr=sys.stderr,\n",
" )\n",
" if out.returncode != 0:\n",
" raise RuntimeError(\"Building binary KenLM was not successful!\")\n",
"\n",
"\n",
"def _get_asr_tokens(audio_filepath, kenlm_bin_dir, transcript_tokens, remove_disfluencies=False):\n",
" with tempfile.TemporaryDirectory() as tmp_dir:\n",
" kenlm_model_filepath = os.path.join(tmp_dir, \"model.bin\")\n",
" _write_kenlm_model(kenlm_bin_dir, transcript_tokens.text, kenlm_model_filepath)\n",
" lm_config = LmConfig(kenlm_model_filepath, unigrams=set(transcript_tokens.text.split()))\n",
" with capture_output():\n",
" # TODO: eliminate warnings by using a script instead of capturing output\n",
" raw_asr_tokens = transcribe(audio_filepath, lm_config=lm_config, mode=\"tokens\")\n",
" # TODO: space inbetween here gets discarded, can we use that?\n",
" asr_tokens = []\n",
" for token_str, (start_s, end_s), time_conf in raw_asr_tokens:\n",
" if len(token_str.strip()) == 0:\n",
" continue\n",
" if remove_disfluencies and token_str in (\"um\", \"uh\"):\n",
" continue\n",
" asr_tokens.append(Token(\n",
" token_str,\n",
" \"text\",\n",
" start_s=start_s,\n",
" end_s=end_s,\n",
" metadata={\"time_conf\": time_conf}\n",
" ))\n",
" asr_tokens = Tokens(asr_tokens)\n",
" wer_score = get_wer(transcript_tokens.text, asr_tokens.text)\n",
" if wer_score > 0.4:\n",
" raise ValueError(\"WER too high\")\n",
" # TODO: test optimal LM params on gdata mate\n",
" print(\"{}% WER\".format(round(100 * wer_score, 1)))\n",
" return asr_tokens\n",
"\n",
"\n",
"kenlm_bin_dir = \"/home/georg/tools/kenlm/build/bin\"\n",
"asr_tokens = _get_asr_tokens(audio_filepath, kenlm_bin_dir, transcript_tokens_norm)\n",
"# 8.9% WER"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "cd236f03",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Token(value='well', type='text', speaker_id=None, start_s=0.44, end_s=0.6, metadata={'time_conf': 0.916}),\n",
" Token(value='get', type='text', speaker_id=None, start_s=0.64, end_s=0.8, metadata={'time_conf': 0.847})]"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"asr_tokens[:2]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "2124d340",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"92.9% alignment fraction\n"
]
}
],
"source": [
"# merge sucessful timestamps into transcript tokens\n",
"from difflib import SequenceMatcher\n",
"\n",
"def _get_timestamped_tokens(transcript_tokens, transcript_tokens_norm, asr_tokens):\n",
" matcher = SequenceMatcher(\n",
" a=[t.value for t in transcript_tokens_norm], \n",
" b=[t.value for t in asr_tokens]\n",
" )\n",
" timestamped_transcript_tokens = [t for t in transcript_tokens]\n",
" for block in matcher.get_matching_blocks():\n",
" if block.size == 0:\n",
" continue\n",
" # TODO: make confidence into an actual time based confidence\n",
" for idx in range(block.size):\n",
" idx_a = block.a + idx\n",
" idx_b = block.b + idx\n",
" orig_idx_a = transcript_tokens_norm[idx_a].metadata[\"idx_orig\"]\n",
" asr_token = asr_tokens[idx_b]\n",
" token = timestamped_transcript_tokens[orig_idx_a]\n",
" token = token.add_timestamps(asr_token.start_s, asr_token.end_s)\n",
" token = token.add_metadata({\"time_conf\": asr_token.metadata[\"time_conf\"]})\n",
" timestamped_transcript_tokens[orig_idx_a] = token\n",
" align_frac = np.mean([t.success for t in timestamped_transcript_tokens if t.type == \"text\"])\n",
" print(\"{}% alignment fraction\".format(round(align_frac * 100, 1)))\n",
" return Tokens(timestamped_transcript_tokens)\n",
"\n",
"\n",
"timestamped_transcript_tokens = _get_timestamped_tokens(transcript_tokens, transcript_tokens_norm, asr_tokens)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "d458331a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Token(value='Well,', type='text', speaker_id='Bevo', start_s=0.44, end_s=0.6, metadata={'time_conf': 0.916}),\n",
" Token(value='good', type='text', speaker_id='Bevo', start_s=None, end_s=None, metadata=None)]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"timestamped_transcript_tokens.tokens[:2]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9f9b9299",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "da3ddcc9",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "96e88596",
"metadata": {},
"source": [
"### use new slicer"
]
},
{
"cell_type": "code",
"execution_count": 42,
"id": "6a619037",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.utils.numbers import safe_round, TIME_GRANULARITY_S\n",
"\n",
"\n",
"class SegmentBoundary:\n",
" def __init__(self, index, start_s, end_s):\n",
" self._index = index\n",
" self._start_s = start_s\n",
" self._end_s = end_s\n",
" \n",
" def __repr__(self):\n",
" return f\"SegmentBoundary(index={self.index}, start_s={self.start_s}, duration_s={self.duration_s})\"\n",
"\n",
" @property\n",
" def index(self):\n",
" return self._index\n",
" \n",
" @property\n",
" def start_s(self):\n",
" return safe_round(self._start_s)\n",
"\n",
" @property\n",
" def end_s(self):\n",
" return safe_round(self._end_s)\n",
"\n",
" @property\n",
" def mid_s(self):\n",
" return safe_round((self._start_s + self._end_s) / 2)\n",
"\n",
" @property\n",
" def duration_s(self):\n",
" return safe_round(self._end_s - self._start_s)\n",
" \n",
"MIN_SILENCE_GAP_S = 0.15\n",
"MIN_CONFIDENCE = 0.4\n",
"\n",
"def _get_break_points(\n",
" transcript_tokens, \n",
" tot_durations_s, \n",
" min_silence_gap_s=MIN_SILENCE_GAP_S, \n",
" min_confidence=MIN_CONFIDENCE,\n",
" ends_are_safe=True,\n",
"):\n",
" # TODO: maybe make 2 layers, first more conservative, then subslice\n",
" break_points = []\n",
" for idx, (l_token, r_token) in enumerate(zip(\n",
" transcript_tokens[:-1], transcript_tokens[1:]\n",
" )):\n",
" if not l_token.success or not r_token.success:\n",
" continue\n",
" # TODO: use whitespace confidence to decide if we are allowed to \n",
" # make a token affine cut without success pad\n",
" if (\n",
" r_token.start_s - l_token.end_s >= min_silence_gap_s and\n",
" l_token.metadata[\"time_conf\"] >= min_confidence and\n",
" r_token.metadata[\"time_conf\"] >= min_confidence\n",
" ):\n",
" gap_start_s = safe_round(l_token.end_s + min_silence_gap_s / 2)\n",
" gap_end_s = safe_round(r_token.start_s - min_silence_gap_s / 2)\n",
" if gap_end_s - gap_start_s > 0:\n",
" break_points.append(SegmentBoundary(idx + 1, gap_start_s, gap_end_s)) \n",
" # add start and end as break point if needed\n",
" if ends_are_safe:\n",
" if len(break_points) == 0 or safe_round(break_points[0].start_s) > TIME_GRANULARITY_S:\n",
" break_points = [SegmentBoundary(0, 0, 0)] + break_points\n",
" if len(break_points) == 0 or safe_round(tot_durations_s - break_points[-1].end_s) > TIME_GRANULARITY_S:\n",
" break_points.append(SegmentBoundary(len(transcript_tokens), tot_durations_s, tot_durations_s)) \n",
" return break_points\n",
"\n",
"\n",
"tot_durations_s = get_audio_properties(audio_filepath)[\"duration_s\"]\n",
"break_points = _get_break_points(timestamped_transcript_tokens, tot_durations_s)"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "813b1aff",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"94.1% coverage\n",
"79.9% coverage (without excluded)\n"
]
}
],
"source": [
"# make segments\n",
"MIN_SLICE_DURATION_S = 2\n",
"MAX_SLICE_DURATION_S = 20\n",
"\n",
"def _get_segments(\n",
" break_points, \n",
" min_slice_duration_s=MIN_SLICE_DURATION_S, \n",
" max_slice_duration_s=MAX_SLICE_DURATION_S,\n",
"):\n",
" # TODO: this algorithm is greedy, a better one with increased coverage likely exists\n",
" # sub-split very long boundaries\n",
" safe_break_points = []\n",
" for break_point in break_points:\n",
" if break_point.duration_s > min_slice_duration_s:\n",
" for n in range(int(break_point.duration_s // min_slice_duration_s + 1)):\n",
" start_s = np.min([break_point.start_s + n * min_slice_duration_s, break_point.end_s])\n",
" safe_break_points.append(SegmentBoundary(break_point.index, start_s, end_s))\n",
" else:\n",
" safe_break_points.append(break_point)\n",
"\n",
" if len(safe_break_points) == 0:\n",
" print(\"0% coverage\")\n",
" return []\n",
" \n",
" cur_start_boundary = safe_break_points[0]\n",
" cur_end_boundary = safe_break_points[0]\n",
" segments = []\n",
" for boundary in safe_break_points:\n",
" # check if latest segment contains excluded token\n",
" segment_tokens = Tokens(timestamped_transcript_tokens[cur_end_boundary.index:boundary.index])\n",
" if segment_tokens.exclude_in_transcript:\n",
" # add current segment if long enough\n",
" if cur_end_boundary.mid_s - cur_start_boundary.mid_s >= min_slice_duration_s:\n",
" segments.append((cur_start_boundary, cur_end_boundary))\n",
" # add excluded slice if right size\n",
" if (\n",
" boundary.mid_s - cur_end_boundary.mid_s >= min_slice_duration_s and\n",
" boundary.start_s - cur_end_boundary.end_s <= max_slice_duration_s\n",
" ):\n",
" segments.append((cur_end_boundary, boundary))\n",
" cur_start_boundary = boundary\n",
" cur_end_boundary = boundary\n",
" continue\n",
" # check if we can add another one to the stack\n",
" if boundary.start_s - cur_start_boundary.end_s <= max_slice_duration_s:\n",
" cur_end_boundary = boundary\n",
" continue\n",
" else:\n",
" # finish current segment\n",
" if cur_end_boundary.mid_s - cur_start_boundary.mid_s >= min_slice_duration_s:\n",
" segments.append((cur_start_boundary, cur_end_boundary))\n",
" # make new current if not too large\n",
" if boundary.start_s - cur_end_boundary.end_s <= max_slice_duration_s:\n",
" cur_start_boundary = cur_end_boundary\n",
" cur_end_boundary = boundary\n",
" else:\n",
" cur_start_boundary = boundary\n",
" cur_end_boundary = boundary\n",
" if cur_end_boundary.mid_s - cur_start_boundary.mid_s >= min_slice_duration_s:\n",
" segments.append((cur_start_boundary, cur_end_boundary))\n",
"\n",
" # pad up to half of breakpoint if possible\n",
" padded_segments = []\n",
" for start_boundary, end_boundary in segments: \n",
" tokens = Tokens(timestamped_transcript_tokens[start_boundary.index:end_boundary.index])\n",
" pad_s = max_slice_duration_s - (end_boundary.start_s - start_boundary.end_s)\n",
" h_pad_s = safe_round(pad_s / 2) - TIME_GRANULARITY_S\n",
" if h_pad_s > 0:\n",
" start_s = np.max([start_boundary.end_s - h_pad_s, start_boundary.mid_s])\n",
" end_s = np.min([end_boundary.start_s + h_pad_s, end_boundary.mid_s])\n",
" padded_segments.append(((start_s, end_s), tokens))\n",
" else:\n",
" padded_segments.append(((start_boundary.end_s, end_boundary.start_s), tokens))\n",
"\n",
" # verify none too small or too large and no overlap\n",
" for (start_s, end_s), _ in padded_segments:\n",
" duration_s = safe_round(end_s - start_s)\n",
" if duration_s < min_slice_duration_s:\n",
" raise ValueError(\"found segment that is too short\")\n",
" if duration_s > max_slice_duration_s:\n",
" raise ValueError(\"found segment that is too long\")\n",
" for ((_, l_end_s), _), ((r_start_s, _), _) in zip(padded_segments[:-1], padded_segments[1:]):\n",
" if safe_round(r_start_s - l_end_s) < 0:\n",
" raise ValueError(\"found overlapping segments\")\n",
"\n",
" cov_frac = np.sum([\n",
" end_s - start_s for (start_s, end_s), _ in padded_segments\n",
" ]) / tot_durations_s\n",
" print(\"{}% coverage\".format(round(cov_frac * 100, 1)))\n",
" cov_frac = np.sum([\n",
" end_s - start_s \n",
" for (start_s, end_s), tokens in padded_segments \n",
" if not tokens.exclude_in_transcript\n",
" ]) / tot_durations_s\n",
" print(\"{}% coverage (without excluded)\".format(round(cov_frac * 100, 1)))\n",
"\n",
" return padded_segments\n",
"\n",
"segments = _get_segments(break_points)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2d7d24e2",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "0b181a6a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "4bda85e6",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "222d96fb",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "7248c30e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "e946d419",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cc04cdf2",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "535eec93",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "39c70927",
"metadata": {},
"source": [
"### investigate alignment"
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "9f8a7d36",
"metadata": {},
"outputs": [],
"source": [
"matcher = SequenceMatcher(a=transcript_tokens, b=asr_tokens)\n",
"mismatched = []\n",
"offs_a = 0\n",
"offs_b = 0\n",
"for block in matcher.get_matching_blocks():\n",
" if block.size >= 1 and (block.a != 0 or block.b != 0):\n",
" mismatched.append(((offs_a, block.a), (offs_b, block.b)))\n",
" offs_a = block.a + block.size\n",
" offs_b = block.b + block.size\n",
" \n",
"for (start_a, end_a), (start_b, end_b) in mismatched:\n",
" print(transcript_tokens[start_a:end_a])\n",
" print(asr_tokens[start_b:end_b])\n",
" print()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "627dde40",
"metadata": {},
"outputs": [],
"source": [
"# TODO ways to improve aligned word fraction:\n",
"# numbers\n",
"# '2018' -> ['twenty', 'eighteen']\n",
"# '24th' -> ['twenty', 'fourth']\n",
"# '19-year-old' -> ['nineteen', 'year', 'old']\n",
"# repetitive substitutions\n",
"# 'curios' -> 'kyrgios'\n",
"# 'ozzies' -> 'aussies'\n",
"# replace small mistakes\n",
"# 'sixes' -> 'sixers'\n",
"# 'test' -> 'tests'\n",
"# 'titles' -> 'totals'\n",
"# ['stark', 'hazewood'] -> ['starc', 'hazelwood']"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "84c470cc",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf9e4195",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "7c67cb71",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "aad330b7",
"metadata": {},
"source": [
"### use existing slicer"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "5dd2f3ab",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"few vad gaps were found, try increasing aggressiveness\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"44.2% coverage\n"
]
}
],
"source": [
"from suno_utils.audio import Audio\n",
"from suno_utils.utils.slicer import get_annotated_segments\n",
"audio = Audio.from_file(audio_filepath)\n",
"segments = get_annotated_segments(\n",
" audio,\n",
" timestamped_transcript_tokens,\n",
" min_gap_s=0.1,\n",
" max_dur_s=20.0,\n",
" safe_pad=1, # TODO: can set this to 0 if we stay gap affine?\n",
" remove_background_for_vad=False, # TODO: maybe we don't even need VAD?\n",
")\n",
"\n",
"cov_frac = sum([a.duration_s for a, _, _ in segments]) / audio.duration_s\n",
"print(\"{}% coverage\".format(round(cov_frac * 100, 1)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "31e7c2cb",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d5b1fbff",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "24e6d9e4",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "41ac1f00",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "d52231bf",
"metadata": {},
"source": [
"### test gap affinity"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "676064e8",
"metadata": {},
"outputs": [],
"source": [
"# Bio.pairwise2 allows gaps to be penalized"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "06baeee6",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['(0, 3), (0, 3): aa bb cc', '(5, 6), (3, 4): ee', '(7, 8), (4, 5): gg']"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tokens_a = \"aa bb cc dd cc ee ff gg\".split()\n",
"tokens_b = \"aa bb cc ee gg\".split()\n",
"matcher = SequenceMatcher(a=tokens_a, b=tokens_b)\n",
"l = []\n",
"for block in matcher.get_matching_blocks():\n",
" if block.size == 0:\n",
" continue\n",
" l.append(\n",
" f\"({block.a}, {block.a+block.size}), ({block.b}, {block.b+block.size}): \" + \n",
" \" \".join(tokens_a[block.a:block.a+block.size])\n",
" )\n",
"l"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f7b94ea0",
"metadata": {},
"outputs": [],
"source": [
"# get op codes with difflib\n",
"# for tag, i1, i2, j1, j2 in matcher.get_opcodes():\n",
"# print(\"%7s a[%d:%d] (%s) b[%d:%d] (%s)\" % (tag, i1, i2, a[i1:i2], j1, j2, b[j1:j2]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b29e8487",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "95a1b54d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d6392217",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "814b8fa5",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e71b78e5",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import subprocess\n",
"\n",
"# write train text\n",
"train_text_filepath = \"/home/georg/notebooks/tasks/segmentation/tmp/text.txt\"\n",
"with open(train_text_filepath, \"w\") as f:\n",
" f.write(\" \".join(transcript_tokens))\n",
"\n",
"# train LM\n",
"ngrams = 5\n",
"model_arpa_filepath = \"/home/georg/notebooks/tasks/segmentation/tmp/model.arpa\"\n",
"train_kenlm_args = [\n",
" \"/home/georg/tools/kenlm/build/bin/lmplz\",\n",
" \"-o\",\n",
" str(ngrams),\n",
" \"--text\",\n",
" train_text_filepath,\n",
" \"--arpa\",\n",
" model_arpa_filepath,\n",
" \"--discount_fallback\",\n",
"]\n",
"out = subprocess.run(train_kenlm_args, capture_output=False, text=True, stdout=sys.stdout, stderr=sys.stderr)\n",
"if out.returncode != 0:\n",
" raise RuntimeError(\"Training KenLM was not successful!\")\n",
"\n",
"# build binary\n",
"model_bin_filepath = \"/home/georg/notebooks/tasks/segmentation/tmp/model.bin\"\n",
"kenlm_args = [\n",
" \"/home/georg/tools/kenlm/build/bin/build_binary\",\n",
" \"-s\",\n",
" \"trie\",\n",
" model_arpa_filepath,\n",
" model_bin_filepath,\n",
"]\n",
"out = subprocess.run(kenlm_args, capture_output=False, text=True, stdout=sys.stdout, stderr=sys.stderr)\n",
"if out.returncode != 0:\n",
" raise RuntimeError(\"Building binary KenLM was not successful!\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e118d11e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "c18bdf06",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "3bac24c9",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f9bc1b2f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 13,
"id": "6cc50c9e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "7a2078a8",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "1e6c20fb",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Exception ignored in: \n",
"Traceback (most recent call last):\n",
" File \"/usr/lib/python3.8/multiprocessing/connection.py\", line 132, in __del__\n",
" self._close()\n",
" File \"/usr/lib/python3.8/multiprocessing/connection.py\", line 361, in _close\n",
" _close(self._handle)\n",
"OSError: [Errno 9] Bad file descriptor\n"
]
}
],
"source": [
"from suno_utils.tasks.asr import LmConfig\n",
"vocab, logits = transcribe(audio_filepath, mode=\"logits\")"
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "a5da1dfc",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(12363, 129)"
]
},
"execution_count": 56,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"logits.shape"
]
},
{
"cell_type": "code",
"execution_count": 179,
"id": "b84ec770",
"metadata": {},
"outputs": [],
"source": [
"from pyctcdecode import build_ctcdecoder\n",
"from suno_utils.utils.display import suppress_logging\n",
"\n",
"with suppress_logging():\n",
" decoder = build_ctcdecoder(\n",
" vocab, \n",
" kenlm_model_path=\"/home/georg/notebooks/tasks/segmentation/tmp/model.bin\",\n",
" unigrams=set(transcript_tokens),\n",
" alpha=0.8,\n",
" beta=2.5,\n",
" unk_score_offset=-10.0,\n",
" lm_score_boundary=False,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 176,
"id": "226f0f57",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'well get everyone we'"
]
},
"execution_count": 176,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# import multiprocessing\n",
"# _, _, tokens_with_frames, _, _ = decoder.decode_beams(logits, beam_width=2)[0]\n",
"# with multiprocessing.Pool(10) as p:\n",
"# pred = decoder.decode_batch(p, logits, beam_width=150)\n",
"pred = decoder.decode(logits, beam_width=150)\n",
"pred[:20]"
]
},
{
"cell_type": "code",
"execution_count": 177,
"id": "20658b3a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.07696102614701529"
]
},
"execution_count": 177,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# 0.183 without\n",
"# 0.077 with LM\n",
"# get_wer(\" \".join(transcript_tokens[:37]), pred)\n",
"get_wer(\" \".join(transcript_tokens), pred)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "277a3712",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "909434df",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "32d043da",
"metadata": {},
"source": [
"### Tune LM"
]
},
{
"cell_type": "code",
"execution_count": 106,
"id": "26eb3589",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
]
},
{
"cell_type": "code",
"execution_count": 107,
"id": "921a3b35",
"metadata": {},
"outputs": [],
"source": [
"import random\n",
"import json\n",
"import tqdm\n",
"import numpy as np\n",
"from suno_utils.web.rev import parse_transcript\n",
"from suno_utils.tasks.ctc_align import _asr_normalize_tokens, _get_asr_tokens, LmAlignConfig\n",
"from suno_utils.utils.metrics import get_wer\n",
"\n",
"with open(\"/mnt/data-ssd-1/data/gdata-mate/pre_pipeline/03_annotation_manifest.json\") as f:\n",
" manifest = json.load(f)\n",
"random.seed(6006)\n",
"random.shuffle(manifest)"
]
},
{
"cell_type": "code",
"execution_count": 110,
"id": "ec55b4ca",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7cf1ff22e61d40baa0ccae2dbd8d4f00",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/10 [00:00 in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:14.7% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:10.9% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:21.0% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:18.9% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:4.7% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:19.7% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:14.0% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:10.8% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:13.0% WER\n",
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n",
"INFO:suno_utils.tasks.ctc_align:9.9% WER\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"sweep_val 2.5\n",
"13.8% mean WER\n"
]
}
],
"source": [
"# DATA_DIR = \"/mnt/data-ssd-1/data/gdata-mate/\"\n",
"# RAW_DATA_DIR = os.path.join(DATA_DIR, \"pre_pipeline/rev_files/\")\n",
"# transcript_filepath = os.path.join(RAW_DATA_DIR, \"3045f69a-52ca-4c35-8e83-7f0df9894889.json\")\n",
"# audio_filepath = os.path.join(RAW_DATA_DIR, \"3045f69a-52ca-4c35-8e83-7f0df9894889.wav\")\n",
"\n",
"metrics = []\n",
"for sweep_val in [2.5]:\n",
" wer_list = []\n",
" for m in tqdm.auto.tqdm(manifest[:10]):\n",
" transcript_filepath = m[\"transcript_filepath\"]\n",
" audio_filepath = m[\"audio_filepath\"]\n",
"\n",
" with open(transcript_filepath) as f:\n",
" transcript = json.load(f)\n",
" transcript_tokens = parse_transcript(transcript)\n",
"\n",
" kenlm_bin_dir = \"/home/georg/tools/kenlm/build/bin\"\n",
"\n",
" lm_config = LmAlignConfig(\n",
" alpha=1.0,\n",
" beta=2.5,\n",
" beam_width=150,\n",
" n_cores=10,\n",
" # unk_score_offset=-10.0,\n",
" # lm_score_boundary=False,\n",
" )\n",
"\n",
" transcript_tokens_norm = _asr_normalize_tokens(transcript_tokens)\n",
" asr_tokens = _get_asr_tokens(\n",
" audio_filepath, transcript_tokens_norm, kenlm_bin_dir=kenlm_bin_dir, lm_config=lm_config,\n",
" asr_n_workers=0,\n",
" )\n",
"\n",
" wer_score = get_wer(transcript_tokens_norm.text, asr_tokens.text)\n",
" wer_list.append(wer_score)\n",
" print(f\"sweep_val {sweep_val}\")\n",
" print(\"{}% mean WER\".format(round(100 * np.mean(wer_list), 1)))\n",
" metrics.append((sweep_val, wer_list))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "645dddfa",
"metadata": {},
"outputs": [],
"source": [
"# with LM\n",
"# 3:50\n",
"# 6.4% mean WER\n",
"\n",
"# no LM\n",
"# 1:00\n",
"# 13.8% mean WER"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "37bed796",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "56e1b01f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "821d3bc5",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 46,
"id": "2fd28046",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.tasks.ctc_align import _asr_normalize_tokens\n",
"transcript_tokens_norm = _asr_normalize_tokens(transcript_tokens)"
]
},
{
"cell_type": "code",
"execution_count": 75,
"id": "c74acb33",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:pyctcdecode.alphabet:Alphabet determined to be of BPE style.\n",
"INFO:pyctcdecode.alphabet:Found in vocabulary, substituting with ▁⁇▁.\n"
]
}
],
"source": [
"from suno_utils.tasks.ctc_align import tempfile, LmAlignConfig, _write_kenlm_model, LmConfig, transcribe\n",
"\n",
"asr_batch_size = 8\n",
"asr_n_workers = 0\n",
"remove_disfluencies = False\n",
"with tempfile.TemporaryDirectory() as tmp_dir:\n",
" kenlm_model_filepath = os.path.join(tmp_dir, \"model.bin\")\n",
" _write_kenlm_model(kenlm_bin_dir, transcript_tokens_norm.text, kenlm_model_filepath)\n",
" lm_config = LmAlignConfig()\n",
" use_lm_config = LmConfig(\n",
" kenlm_model_filepath,\n",
" unigrams=set(transcript_tokens_norm.text.split()),\n",
" alpha=lm_config.alpha,\n",
" beta=lm_config.beta,\n",
" beam_width=lm_config.beam_width,\n",
" n_cores=lm_config.n_cores,\n",
" unk_score_offset=lm_config.unk_score_offset,\n",
" lm_score_boundary=lm_config.lm_score_boundary,\n",
" )\n",
" raw_asr_tokens = transcribe(\n",
" audio_filepath,\n",
" lm_config=None,#use_lm_config,\n",
" mode=\"tokens\",\n",
" batch_size=asr_batch_size,\n",
" n_workers=asr_n_workers,\n",
" quiet=True,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 76,
"id": "cbb0f29a",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.tasks.ctc_align import Token, Tokens\n",
"# TODO: space in-between here gets discarded, can we use that?\n",
"asr_tokens = []\n",
"for token_str, (start_s, end_s), time_conf in raw_asr_tokens:\n",
" if len(token_str.strip()) == 0:\n",
" continue\n",
" if remove_disfluencies and token_str in (\"um\", \"uh\"):\n",
" continue\n",
" asr_tokens.append(Token(\n",
" token_str,\n",
" \"text\",\n",
" start_s=start_s,\n",
" end_s=end_s,\n",
" metadata={\"time_conf\": time_conf}\n",
" ))\n",
"asr_tokens = Tokens(asr_tokens)"
]
},
{
"cell_type": "code",
"execution_count": 77,
"id": "ded6b6ab",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Token(value='an', type='text', speaker_id=None, start_s=414.84, end_s=414.92, metadata={'time_conf': 0.958}),\n",
" Token(value='absolute', type='text', speaker_id=None, start_s=414.92, end_s=415.36, metadata={'time_conf': 0.907}),\n",
" Token(value='champion', type='text', speaker_id=None, start_s=415.44, end_s=415.72, metadata={'time_conf': 0.986}),\n",
" Token(value='so', type='text', speaker_id=None, start_s=415.84, end_s=415.92, metadata={'time_conf': 0.98}),\n",
" Token(value='yeah', type='text', speaker_id=None, start_s=416.36, end_s=416.4, metadata={'time_conf': 0.966}),\n",
" Token(value='i', type='text', speaker_id=None, start_s=416.6, end_s=416.68, metadata={'time_conf': 0.953}),\n",
" Token(value='think', type='text', speaker_id=None, start_s=416.76, end_s=416.88, metadata={'time_conf': 0.994}),\n",
" Token(value=\"it's\", type='text', speaker_id=None, start_s=416.88, end_s=417.04, metadata={'time_conf': 0.984}),\n",
" Token(value='um', type='text', speaker_id=None, start_s=417.08, end_s=417.24, metadata={'time_conf': 0.857}),\n",
" Token(value=\"it's\", type='text', speaker_id=None, start_s=417.28, end_s=417.44, metadata={'time_conf': 0.963})]"
]
},
"execution_count": 77,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"nn = [n for n, t in enumerate(asr_tokens) if t.end_s <= 416][-1]\n",
"asr_tokens[nn-3:nn+7]"
]
},
{
"cell_type": "code",
"execution_count": 83,
"id": "67e5cd52",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[(' ', (415.36, 415.44), 0.971),\n",
" ('champion', (415.44, 415.72), 0.986),\n",
" (' ', (415.72, 415.84), 0.993),\n",
" ('so', (415.84, 415.92), 0.98),\n",
" (' ', (415.92, 416.36), 1.0),\n",
" ('yeah', (416.36, 416.4), 0.966),\n",
" (' ', (416.4, 416.6), 0.989),\n",
" ('i', (416.6, 416.68), 0.953),\n",
" (' ', (416.68, 416.76), 0.991),\n",
" ('think', (416.76, 416.88), 0.994)]"
]
},
"execution_count": 83,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"nn = [n for n, (_, (_, t), _) in enumerate(raw_asr_tokens) if t <= 416][-1]\n",
"raw_asr_tokens[nn-3:nn+7]"
]
},
{
"cell_type": "code",
"execution_count": 73,
"id": "6eb52004",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"audio.get_slice(416.36, 416.7).play()"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "011a1a25",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:suno_utils.tasks.ctc_align:92.5% alignment fraction\n"
]
}
],
"source": [
"from suno_utils.tasks.ctc_align import _get_timestamped_tokens\n",
"out_timestamped_transcript_tokens = _get_timestamped_tokens(\n",
" transcript_tokens, transcript_tokens_norm, asr_tokens,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "59c1a0f1",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1997"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(asr_tokens)"
]
},
{
"cell_type": "code",
"execution_count": 48,
"id": "f6ee72ee",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"2027"
]
},
"execution_count": 48,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(transcript_tokens_norm)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b26efffa",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "209e7d8e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "44b13ec3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "7800e897",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}