{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "5e5e22f5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "c083f71f",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ['CUDA_VISIBLE_DEVICES'] = ''"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "9e0547b7",
"metadata": {},
"outputs": [],
"source": [
"import re\n",
"\n",
"from suno_utils.utils.conversion import Audio\n",
"from suno_utils.utils.parser import filter_lines, get_pdf_lines, parse_transcript_segments, segments_to_tokens\n",
"from suno_utils.utils.slicer import get_annotated_slices\n",
"from suno_utils.utils.text import normalize_whitespace\n",
"from suno_utils.web.gentle import align_transcript, postprocess_align_dict\n",
"\n",
"from suno_utils.utils.notebook import _"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d1ef2bc3",
"metadata": {},
"outputs": [],
"source": [
"def _format_segments(segments):\n",
" \"\"\"Corpus specific basic formatting of text.\"\"\"\n",
" clean_segments = []\n",
" for segment_type, speaker, text in segments:\n",
" if segment_type == \"speech\":\n",
" # simplify hesitations\n",
" clean_text = text.replace(\"--\", \" -- \").replace(\"...\", \" -- \")\n",
" # turn parens sidenotes into commas\n",
" # TODO: maybe we should turn this into -- ?\n",
" clean_text = re.sub(r\"\\s*\\(\\s*(.+?)\\s*\\)\", \", \\\\1\", clean_text)\n",
" clean_text = normalize_whitespace(clean_text)\n",
" for p in re.split(r\"\\[(.+?)\\]|(\\-\\-)\", clean_text):\n",
" if p is None or len(p.strip()) == 0:\n",
" continue\n",
" p = p.strip()\n",
" if p == \"--\":\n",
" clean_segments.append((\"metatag\", None, \"hesitation\"))\n",
" else:\n",
" clean_segments.append((segment_type, speaker, p))\n",
" elif segment_type == \"metatag\":\n",
" clean_segments.append((segment_type, None, text.strip(\"[]\")))\n",
" else:\n",
" raise NotImplementedError(\"unknown segment type\")\n",
" return clean_segments"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "ca58231e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"87.1% aligned\n",
"90.2% slice coverage\n"
]
}
],
"source": [
"pdf_fp = \"brainscience.pdf\"\n",
"audio_fp = \"brainscience.mp3\"\n",
"\n",
"# load and parse transcript\n",
"raw_lines = get_pdf_lines(pdf_fp)\n",
"start_ptn = r\"INTERVIEW\"\n",
"end_ptn = r\"Brain Science with Dr\\. Ginger Campbell is copyright\"\n",
"filtered_lines = filter_lines(raw_lines, start_ptn, end_ptn)\n",
"speaker_ptn = r\"^[^0-9]+\\:\"\n",
"segments = parse_transcript_segments(filtered_lines, speaker_ptn=speaker_ptn)\n",
"segments = _format_segments(segments)\n",
"speaker_names, tokens = segments_to_tokens(segments, included_tags=[\"hesitation\"])\n",
"\n",
"# load audio\n",
"audio = Audio.from_file(audio_fp, n_channels=1)\n",
"audio_16khz = audio.convert(16_000, 2, 1)\n",
"\n",
"# align audio & transcript\n",
"fulltext = \" \".join([token[\"value\"] for token in tokens if token[\"type\"] == \"text\"])\n",
"align_dict = align_transcript(audio_16khz, fulltext)\n",
"align_meta = postprocess_align_dict(align_dict, tokens_with_tags=tokens)\n",
"print(\"{}% aligned\".format(round(np.mean([e[\"success\"] for e in align_meta]) * 100, 1)))\n",
"\n",
"# make slices\n",
"slices = get_annotated_slices(audio, align_meta)\n",
"cov_frac = np.sum([end_s - start_s for _, (start_s, end_s), _ in slices]) / audio.duration_s\n",
"print(\"{}% slice coverage\".format(round(cov_frac * 100, 1)))\n",
"\n",
"# 87.1% aligned\n",
"# 90.2% slice coverage"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a6c4926",
"metadata": {},
"outputs": [],
"source": [
"# TODO: nemo preds, slice filtering, montreal word alignment and normalization"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "473d2345",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"31\n",
"[(1, 'You can say, \\'We\\'ll drink beer,\" and that is enough to understand that something is going on with beer-drinking; but in order to really')]\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"n = random.choice(list(range(len(slices))))\n",
"a, _, t = slices[n]\n",
"print(n)\n",
"print(t)\n",
"a.play()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1cab1ce6",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "fff3ed1a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "47e4a9df",
"metadata": {},
"outputs": [],
"source": [
"## Formatting study\n",
"# parentheses are side-notes\n",
"# trailing off vs hesitation: two minutes...\n",
"# \"am gonna sorta try\" -> am going to try\n",
"# same for \"sort of\" but not always\n",
"# \"i wanna\" - > I want to\n",
"# \"okay, you\" - > You\n",
"# \"right, and i\" - > And I\n",
"# \"the error -- that is the brain\" - > the error -- the brain\n",
"# \"like\" removed"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8aea330f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cb251e51",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "36e72f92",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "31c4b08f",
"metadata": {},
"outputs": [],
"source": [
"# TODO: the below should vanish\n",
"def postprocess_slice_meta(\n",
" slice_intervals,\n",
" align_meta,\n",
" fulltext,\n",
" speaker_idx_map,\n",
" meta_idx_map,\n",
" meta_ptn=DEFAULT_META_PTN,\n",
" min_align_frac=0.6,\n",
"):\n",
" # annotate with text and speakers\n",
" slice_meta = [] # [(start, end), turns]\n",
" for start_s, end_s in slice_intervals:\n",
" # get character index spans\n",
" align_idx_start = None\n",
" align_idx_end = None\n",
" char_idx_start = None\n",
" char_idx_end = None\n",
" for n, e in enumerate(align_meta):\n",
" if \"start_s\" in e and e[\"start_s\"] >= start_s and char_idx_start is None:\n",
" align_idx_start = n\n",
" char_idx_start = e[\"startOffset\"]\n",
" if \"end_s\" in e and e[\"end_s\"] <= end_s:\n",
" align_idx_end = n + 1\n",
" char_idx_end = e[\"endOffset\"]\n",
"\n",
" # one can be none if edge but not both\n",
" if char_idx_start is None and char_idx_end is None:\n",
" raise ValueError(\"both start and end index are None\")\n",
" # assemble speaker segments within index range (if one is none then empty segments)\n",
" speaker_turns = []\n",
" if char_idx_start is not None and char_idx_end is not None:\n",
" # check min align fraction in slice\n",
" slice_align_frac = (\n",
" 1.0\n",
" if len(align_meta[align_idx_start:align_idx_end]) == 0\n",
" else np.mean([\n",
" e[\"success\"] for e in align_meta[align_idx_start:align_idx_end]\n",
" ])\n",
" )\n",
" if slice_align_frac < min_align_frac:\n",
" # skip slice since unreliably aligned\n",
" continue\n",
" # results all intervals per speaker so need to separate and sort\n",
" interval_container = []\n",
" for (\n",
" intervals, speaker,\n",
" ) in speaker_idx_map[portion.closedopen(char_idx_start, char_idx_end)].items():\n",
" for interval in intervals:\n",
" interval_container.append((interval, speaker))\n",
" interval_container = sorted(interval_container, key=lambda x: x[0].lower)\n",
" for interval, speaker in interval_container:\n",
" text = _reconstitute_slice_text(\n",
" interval.lower, interval.upper, fulltext, meta_idx_map\n",
" )\n",
" for n, sub_text in enumerate(re.split(meta_ptn, text)):\n",
" sub_text = normalize_whitespace(sub_text)\n",
" if n % 2 == 0 and len(sub_text) > 0:\n",
" speaker_turns.append((speaker, sub_text))\n",
" elif n % 2 == 1:\n",
" speaker_turns.append((None, sub_text))\n",
" if (\n",
" len(speaker_turns) >= 1 and\n",
" (\n",
" re.match(meta_ptn, speaker_turns[0][-1]) or\n",
" re.match(meta_ptn, speaker_turns[-1][-1])\n",
" )\n",
" ):\n",
" # meta tags at boundaries are unreliable in timing\n",
" continue\n",
" slice_meta.append(((round(start_s, 2), round(end_s, 2)), speaker_turns))\n",
" return slice_meta\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "653d654b",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}