{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"id": "b304dfff",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "f955e852",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "e9716000",
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"import os\n",
"import tqdm\n",
"import shutil\n",
"import random\n",
"import funcy\n",
"import json\n",
"import uuid\n",
"import numpy as np\n",
"import multiprocessing\n",
"import pandas as pd\n",
"\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.audio import Tokens\n",
"\n",
"\n",
"BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/\"\n",
"\n",
"DATE_DIR = \"2022_08_11\"\n",
"SEGMENT_DATE_DIR = \"2022_08_11_2\"\n",
"\n",
"ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"artifacts\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ad5c3cc7",
"metadata": {},
"outputs": [],
"source": [
"# \"call you at 12 tomorrow\" and the normalized was \"call you at twelve oh oh tomorrow\"."
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "33460c02",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"48.5 hours\n"
]
}
],
"source": [
"with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n",
" segments_manifest = json.load(f)\n",
"print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "e611168a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"6 samples found.\n",
"----------\n",
"9e249c3c-6799-4004-9f83-5a0b175025c2\n",
"Hm, okay, yeah. 8:00. Thank you so much, sir, that info. So yeah, [inaudible] a while, uh, let me checking for that issue, okay?\n",
"hm okay yeah eight oh oh thank you so much sir that info so yeah a while uh let me checking for that issue okay\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"d1d863ef-27e2-4e69-9932-e62ceea87b9c\n",
"Uh, because here in the Philippines it's, uh, 6:00 A -- Uh, almost 6:00 AM.\n",
"uh because here in the philippines it's uh six oh oh a uh almost six am\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n",
"043f949d-fb26-4e99-9133-27c4e0ac57d3\n",
"I started to work at 9:00, uh, 9:00 PM.\n",
"i started to work at nine oh oh uh nine pm\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------\n"
]
}
],
"source": [
"l = [\n",
" e for e in segments_manifest \n",
" if (\n",
" re.search(r\"[0-9]\\:[0-9]\", Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text) and\n",
" \"oh oh\" in Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text\n",
" )\n",
"]\n",
"print(len(l), \"samples found.\")\n",
"print(\"-\"*10)\n",
"for e in l[:3]:\n",
" print(e[\"id\"])\n",
" print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n",
" print(Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n",
" Audio.from_file(e[\"uri\"]).play()\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "62b17399",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"I started to work at 9:00, uh, 9:00 PM.\n",
"i started to work at nine oh oh uh nine pm\n",
"i started work at nine uh nine p m\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"m = [\n",
" e for e in segments_manifest \n",
" if e[\"id\"] == \"043f949d-fb26-4e99-9133-27c4e0ac57d3\"\n",
"][0]\n",
"print(Tokens.from_dict(m[\"transcript\"][\"tokens\"]).text)\n",
"print(Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).text)\n",
"print(m[\"transcript_asr\"][\"transcript\"])\n",
"Audio.from_file(m[\"uri\"]).play()"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "ff451971",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.utils.text_normalizer import NORMALIZER, NORMALIZER_DETERMINISTIC, _pre_normalize, _normalize"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "a587edf6",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'nine PM'"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"NORMALIZER_DETERMINISTIC.normalize(\n",
" text=\"9:00 pm\",\n",
" verbose=False,\n",
" punct_post_process=False,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "da4fc4b7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"9:00\n"
]
},
{
"data": {
"text/plain": [
"{\"nine o'clock\", 'nine: oh oh', 'nine: oh zero', 'nine: zero zero'}"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"NORMALIZER.normalize(\n",
" text=\"9:00\",\n",
" verbose=False,\n",
" n_tagged=-1,\n",
" punct_post_process=False,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "df68a86b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'a 9 b'"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"re.sub(r\"\\s*\\b([12]?[0-9]+)\\:00\\b\\s*\", \" \\\\1 \", \"a 9:00 b\")"
]
},
{
"cell_type": "code",
"execution_count": 39,
"id": "cddcd985",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[\"thirteen o'clock\"]"
]
},
"execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"custom_norm_rules = [\n",
" (re.compile(r\"\\b([12]?[0-9]+)\\:00\\b\"), \" \\\\1 \"), # URLs\n",
"]\n",
"\n",
"_normalize(\n",
" text_list=[\"13:00\"],\n",
" asr_prediction=\"one\",\n",
" custom_norm_rules=custom_norm_rules,\n",
" max_allowed_cer=0.5,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "e260ef5b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"13:00\n"
]
},
{
"data": {
"text/plain": [
"{'one three: oh oh',\n",
" 'one three: oh zero',\n",
" 'one three: zero zero',\n",
" \"thirteen o'clock\",\n",
" 'thirteen: oh oh',\n",
" 'thirteen: oh zero',\n",
" 'thirteen: zero zero'}"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"NORMALIZER.normalize(\n",
" text=\"13:00\",\n",
" verbose=False,\n",
" n_tagged=-1,\n",
" punct_post_process=False,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "c6c021db",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"9:00 pm\n"
]
},
{
"data": {
"text/plain": [
"{'nine PM',\n",
" \"nine o'clock pm\",\n",
" 'nine: oh oh pm',\n",
" 'nine: oh zero pm',\n",
" 'nine: zero PM',\n",
" 'nine: zero zero pm'}"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"NORMALIZER.normalize(\n",
" text=\"9:00 pm\",\n",
" verbose=False,\n",
" n_tagged=-1,\n",
" punct_post_process=False,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "0db87629",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"I started to work at 9:00, uh, 9:00 PM.\n"
]
},
{
"data": {
"text/plain": [
"{\"I started to work at nine o'clock, uh, nine PM.\",\n",
" \"I started to work at nine o'clock, uh, nine o'clock P M.\",\n",
" \"I started to work at nine o'clock, uh, nine o'clock PM.\",\n",
" \"I started to work at nine o'clock, uh, nine: oh oh P M.\",\n",
" \"I started to work at nine o'clock, uh, nine: oh oh PM.\",\n",
" \"I started to work at nine o'clock, uh, nine: oh zero P M.\",\n",
" \"I started to work at nine o'clock, uh, nine: oh zero PM.\",\n",
" \"I started to work at nine o'clock, uh, nine: zero PM.\",\n",
" \"I started to work at nine o'clock, uh, nine: zero zero P M.\",\n",
" \"I started to work at nine o'clock, uh, nine: zero zero PM.\",\n",
" 'I started to work at nine: oh oh, uh, nine PM.',\n",
" \"I started to work at nine: oh oh, uh, nine o'clock P M.\",\n",
" \"I started to work at nine: oh oh, uh, nine o'clock PM.\",\n",
" 'I started to work at nine: oh oh, uh, nine: oh oh P M.',\n",
" 'I started to work at nine: oh oh, uh, nine: oh oh PM.',\n",
" 'I started to work at nine: oh oh, uh, nine: oh zero P M.',\n",
" 'I started to work at nine: oh oh, uh, nine: oh zero PM.',\n",
" 'I started to work at nine: oh oh, uh, nine: zero PM.',\n",
" 'I started to work at nine: oh oh, uh, nine: zero zero P M.',\n",
" 'I started to work at nine: oh oh, uh, nine: zero zero PM.',\n",
" 'I started to work at nine: oh zero, uh, nine PM.',\n",
" \"I started to work at nine: oh zero, uh, nine o'clock P M.\",\n",
" \"I started to work at nine: oh zero, uh, nine o'clock PM.\",\n",
" 'I started to work at nine: oh zero, uh, nine: oh oh P M.',\n",
" 'I started to work at nine: oh zero, uh, nine: oh oh PM.',\n",
" 'I started to work at nine: oh zero, uh, nine: oh zero P M.',\n",
" 'I started to work at nine: oh zero, uh, nine: oh zero PM.',\n",
" 'I started to work at nine: oh zero, uh, nine: zero PM.',\n",
" 'I started to work at nine: oh zero, uh, nine: zero zero P M.',\n",
" 'I started to work at nine: oh zero, uh, nine: zero zero PM.',\n",
" 'I started to work at nine: zero zero, uh, nine PM.',\n",
" \"I started to work at nine: zero zero, uh, nine o'clock P M.\",\n",
" \"I started to work at nine: zero zero, uh, nine o'clock PM.\",\n",
" 'I started to work at nine: zero zero, uh, nine: oh oh P M.',\n",
" 'I started to work at nine: zero zero, uh, nine: oh oh PM.',\n",
" 'I started to work at nine: zero zero, uh, nine: oh zero P M.',\n",
" 'I started to work at nine: zero zero, uh, nine: oh zero PM.',\n",
" 'I started to work at nine: zero zero, uh, nine: zero PM.',\n",
" 'I started to work at nine: zero zero, uh, nine: zero zero P M.',\n",
" 'I started to work at nine: zero zero, uh, nine: zero zero PM.'}"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"text = \"I started to work at 9:00, uh, 9:00 PM.\"\n",
"\n",
"NORMALIZER.normalize(\n",
" text=_pre_normalize(text),\n",
" verbose=False,\n",
" n_tagged=-1,\n",
" punct_post_process=False,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "59bd974b",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "122f92da",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "249e42ef",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "fb8a65f2",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5bd2c28b",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "bb648c97",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}