{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "bf70f34f", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import json\n", "import tqdm\n", "import uuid\n", "import funcy\n", "import numpy as np\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.conversion import get_audio_properties, play_audio\n", "from suno_utils.utils.numbers import safe_round\n", "\n", "BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/speechly/2022-08-30_gymclass_1h\"\n", "RAW_AUDIO_DIR = os.path.join(BASE_DIR, \"raw_audio\")\n", "\n", "AUDIO_DIR = os.path.join(BASE_DIR, \"audio\")\n", "TO_REV_DIR = os.path.join(BASE_DIR, \"to_rev\")\n", "TO_REV_AUDIO_DIR = os.path.join(TO_REV_DIR, \"audio\")\n", "FROM_REV_DIR = os.path.join(BASE_DIR, \"from_rev\")\n", "\n", "SEGMENT_DATE_DIR = \"2022_08_30\"\n", "SEGMENTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"segments\")\n", "ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"artifacts\")" ] }, { "cell_type": "code", "execution_count": 2, "id": "c769f21a", "metadata": {}, "outputs": [], "source": [ "os.makedirs(AUDIO_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_DIR, exist_ok=True)\n", "os.makedirs(TO_REV_AUDIO_DIR, exist_ok=True)\n", "os.makedirs(FROM_REV_DIR, exist_ok=True)\n", "os.makedirs(SEGMENTS_DIR, exist_ok=True)\n", "os.makedirs(ARTIFACTS_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 3, "id": "c3cd79ed", "metadata": {}, "outputs": [], "source": [ "GLOBAL_HOTWORDS = [\n", " 'beta lobby',\n", " 'beta physics',\n", " 'boom discord ball',\n", " 'bozo',\n", " 'carpet muncher',\n", " 'coke butter cheese',\n", " 'comp rules',\n", " 'dadass',\n", " 'deadass',\n", " 'dribble',\n", " 'dunk competition',\n", " 'dunk dink dunk',\n", " 'dunk on dunk',\n", " 'faggy',\n", " 'fcuk',\n", " 'freaking weirdo',\n", " 'fuckulus',\n", " 'gorilla tag mid',\n", " 'green bean',\n", " 'high glitch',\n", " 'invisible wall',\n", " 'red skin',\n", " 'rim',\n", " 'watermelon munchers',\n", " 'you sped',\n", "]" ] }, { "cell_type": "markdown", "id": "3bf11bab", "metadata": {}, "source": [ "## Prepare rev data" ] }, { "cell_type": "code", "execution_count": 14, "id": "9ff97b1c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 644/644 [00:14<00:00, 44.01it/s]\n" ] } ], "source": [ "SAMPLE_RATE = 16_000\n", "\n", "filepaths_info = []\n", "for fn in tqdm.tqdm(os.listdir(RAW_AUDIO_DIR)):\n", " if fn.split(\".\")[-1] != \"wav\":\n", " continue\n", " file_id = fn[:-4]\n", " assert(len(file_id) == 36)\n", " uid = file_id\n", " from_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " to_fp = os.path.join(RAW_AUDIO_DIR, fn)\n", " audio = Audio.from_file(os.path.join(RAW_AUDIO_DIR, fn), sample_rate=SAMPLE_RATE, byte_width=2)\n", " to_filepath = os.path.join(AUDIO_DIR, uid + \".mp3\")\n", " audio.to_mp3(to_filepath)\n", " filepaths_info.append((uid, from_fp, to_filepath))" ] }, { "cell_type": "code", "execution_count": 20, "id": "deb0bebc", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"raw_data_manifest.json\"), \"w\") as f:\n", " json.dump(filepaths_info, f)" ] }, { "cell_type": "code", "execution_count": 21, "id": "4da818da", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "9.7 mins of audio\n", "10.8 mins of audio\n", "10.5 mins of audio\n", "11.1 mins of audio\n", "9.8 mins of audio\n", "10.8 mins of audio\n", "4.9 mins of audio\n" ] } ], "source": [ "CHUNKSIZE = 100\n", "INDICATOR_AUDIO_DIR = \"/home/georg/notebooks/customers/sanas/audio_indicators/\"\n", "\n", "audio_rev_metas = []\n", "for filepaths_chunk in funcy.chunks(CHUNKSIZE, filepaths_info):\n", " audio_segments = []\n", " segments_meta = []\n", " for n, (uid, _, filepath) in enumerate(filepaths_chunk):\n", " indicator_str = str(n).zfill(2)\n", " indicator_duration_s = 0\n", " for c in indicator_str:\n", " indicator_audio = Audio.from_file(\n", " os.path.join(INDICATOR_AUDIO_DIR, f\"{c}_fast.wav\"), sample_rate=SAMPLE_RATE, byte_width=2,\n", " )\n", " indicator_duration_s += indicator_audio.duration_s\n", " audio_segments.append(indicator_audio)\n", " segments_meta.append({\n", " \"segment_number\": n,\n", " \"type\": \"indicator\",\n", " \"duration_s\": safe_round(indicator_duration_s),\n", " \"indicator_str\": indicator_str,\n", " })\n", " audio = Audio.from_file(filepath, sample_rate=SAMPLE_RATE, byte_width=2)\n", " audio_segments.append(audio)\n", " segments_meta.append({\n", " \"segment_number\": n,\n", " \"type\": \"speech\",\n", " \"duration_s\": safe_round(audio.duration_s),\n", " \"orginial_audio_offset_s\": 0,\n", " \"orginial_audio_uid\": uid,\n", " })\n", " audio_rev = Audio.concatenate(audio_segments)\n", " print(round(audio_rev.duration_s / 60, 1), \"mins of audio\") \n", " uid = str(uuid.uuid4())\n", " rev_audio_filepath = os.path.join(TO_REV_AUDIO_DIR, f\"{uid}.mp3\")\n", " audio_rev.to_mp3(rev_audio_filepath)\n", " audio_rev_metas.append({\n", " \"uid\": uid,\n", " \"audio_filepath\": rev_audio_filepath,\n", " \"segments_meta\": segments_meta,\n", " \"duration_s\": audio_rev.duration_s,\n", " })\n", " \n", "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\"), \"w\") as f:\n", " json.dump(audio_rev_metas, f)" ] }, { "cell_type": "code", "execution_count": 37, "id": "b408f4f7", "metadata": {}, "outputs": [], "source": [ "# from suno_utils.tasks.music import remove_music\n", "# audio_rev_vocals = remove_music(audio_rev, wet=1.0, model_name=\"mdx_extra_q\")" ] }, { "cell_type": "code", "execution_count": null, "id": "b17cda8c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dfb40afb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0e26b214", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dfc212cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "88a9e9cc", "metadata": {}, "source": [ "## Send stuff to rev" ] }, { "cell_type": "code", "execution_count": 28, "id": "05fbe672", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, upload_file, make_order, make_cc_order\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 29, "id": "2db71de1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "67.6 minutes total\n" ] } ], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)\n", "\n", "ORDER_REF_STR = \"sl_gc_1h\"\n", "\n", "tot_duration_s = np.sum([m[\"duration_s\"] for m in audio_rev_metas])\n", "print(round(tot_duration_s / 60, 1), \"minutes total\")" ] }, { "cell_type": "code", "execution_count": 31, "id": "8c66f7f9", "metadata": {}, "outputs": [], "source": [ "order_items = []\n", "for m in tqdm.tqdm(audio_rev_metas):\n", " media_loc = upload_file(AUTH_STR, m[\"audio_filepath\"], file_ref_str=m[\"uid\"])\n", " order_items.append(\n", " {\n", " \"media_loc\": media_loc,\n", " \"hotwords\": GLOBAL_HOTWORDS,\n", " \"duration_s\": int(np.ceil(m[\"duration_s\"])),\n", " \"uuid\": m[\"uid\"],\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": 32, "id": "a0ee5f4f", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"w\") as f:\n", " json.dump(order_items, f)" ] }, { "cell_type": "code", "execution_count": 33, "id": "8318e46f", "metadata": {}, "outputs": [], "source": [ "# order_number = make_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)\n", "order_number = make_cc_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR)" ] }, { "cell_type": "code", "execution_count": 34, "id": "27549d51", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\"), \"w\") as f:\n", " f.write(order_number)" ] }, { "cell_type": "code", "execution_count": null, "id": "0096715b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "576b2bef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b33dc8e0", "metadata": {}, "source": [ "## Get rev stuff" ] }, { "cell_type": "code", "execution_count": 38, "id": "e8b59ece", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string, get_finished_order, get_transcript\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", " \n", "CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n", "USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n", "AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)" ] }, { "cell_type": "code", "execution_count": 39, "id": "9e4e40af", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\")) as f:\n", " order_number = f.read()" ] }, { "cell_type": "code", "execution_count": 40, "id": "d2a371b5", "metadata": {}, "outputs": [], "source": [ "# # grab while still in progress\n", "# from suno_utils.web.rev import _get_order_details, _parse_order_response\n", "# order_details = _get_order_details(AUTH_STR, order_number)\n", "# _, transcript_metas = _parse_order_response(order_details)" ] }, { "cell_type": "code", "execution_count": 56, "id": "9275ae0d", "metadata": {}, "outputs": [], "source": [ "# grab all of it\n", "transcript_metas = get_finished_order(AUTH_STR, order_number, blocking=False)" ] }, { "cell_type": "code", "execution_count": 80, "id": "df28d887", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 6.27it/s]\n" ] } ], "source": [ "for transcript_meta in tqdm.tqdm(transcript_metas):\n", " transcript_filepath = os.path.join(FROM_REV_DIR, transcript_meta[\"uuid\"] + \".srt\")\n", " if os.path.exists(transcript_filepath):\n", " continue\n", " transcript_subrip = get_transcript(AUTH_STR, transcript_meta)\n", " with open(transcript_filepath, \"w\") as f:\n", " f.write(transcript_subrip)" ] }, { "cell_type": "code", "execution_count": 82, "id": "ed8a84c6", "metadata": {}, "outputs": [], "source": [ "# !rm /mnt/data-ssd-1/data/private/customer/speechly/2022-08-30_gymclass_1h/from_rev/acc78c33-25fa-4d16-9e02-6c4a74f5b373.srt" ] }, { "cell_type": "code", "execution_count": null, "id": "6f516c73", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f2b631bc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "110b1246", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b15ce9de", "metadata": {}, "source": [ "## Parsing rev subrips" ] }, { "cell_type": "code", "execution_count": 4, "id": "2b80ff9a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-09-01 13:08:37 optimizers:77] Could not import distributed_fused_adam optimizer from Apex\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[NeMo I 2022-09-01 13:08:38 post_processing:50] Post processing graph was restored from /tmp/prod_nemo_fst_georg/en_tn_post_processing.far.\n", "[NeMo I 2022-09-01 13:08:46 tokenize_and_classify_with_audio:99] ClassifyFst.fst was restored from /tmp/prod_nemo_fst_georg/_cased_en_tn_False_deterministicnemo_custom_norm.tsv.far.\n", "[NeMo I 2022-09-01 13:08:46 verbalize_final:52] VerbalizeFinalFst graph was restored from /tmp/prod_nemo_fst_georg/en_tn_False_deterministic_verbalizer.far.\n", "[NeMo I 2022-09-01 13:08:46 post_processing:50] Post processing graph was restored from /tmp/prod_nemo_fst_2_georg/en_tn_post_processing.far.\n", "[NeMo I 2022-09-01 13:08:46 tokenize_and_classify:85] ClassifyFst.fst was restored from /tmp/prod_nemo_fst_2_georg/en_tn_True_deterministic_cased_nemo_custom_norm.tsv_tokenize.far.\n", "[NeMo I 2022-09-01 13:08:46 verbalize_final:52] VerbalizeFinalFst graph was restored from /tmp/prod_nemo_fst_2_georg/en_tn_True_deterministic_verbalizer.far.\n" ] } ], "source": [ "from nltk.corpus import words as nltk_words\n", "from suno_utils.utils.numbers import safe_round\n", "from suno_utils.audio.data_model import Token, Tokens, TEXT, TAG\n", "from suno_utils.customers.sanas.pipeline import get_segments\n", "from suno_utils.utils.text import normalize_whitespace\n", "\n", "COMMON_EN_WORDS = set(nltk_words.words())\n", "\n", "def _parse_srt(transcript_srt):\n", " blocks = []\n", " for block in transcript_srt.strip().split(\"\\n\\n\"):\n", " lines = block.strip().split(\"\\n\")\n", " segment_nr = int(lines[0])\n", " start_str, end_str = lines[1].split(\" --> \")\n", " h, m, s = start_str.split(\":\")\n", " start_s = int(h) * 60**2 + int(m) * 60 + float(s.replace(\",\", \".\"))\n", " h, m, s = end_str.split(\":\")\n", " end_s = int(h) * 60**2 + int(m) * 60 + float(s.replace(\",\", \".\"))\n", " text_lines = lines[2:]\n", " blocks.append({\n", " \"segment_nr\": segment_nr, \n", " \"offset_s\": safe_round(start_s),\n", " \"duration_s\": safe_round(end_s - start_s),\n", " \"text_lines\": text_lines, \n", " })\n", " if blocks[0][\"segment_nr\"] != 1:\n", " raise ValueError(\"First block doesn't start with 1.\")\n", " if any(np.diff([e[\"segment_nr\"] for e in blocks]) != 1):\n", " raise ValueError(\"Parsed block numbers are not consecutive.\")\n", " return blocks\n", "\n", "def _tag_repl(m):\n", " s = m.group()\n", " if s is not None:\n", " s = s.lower()\n", " s = re.sub(r\"[\\(\\)]\", \" \", s.lower())\n", " s = normalize_whitespace(s)\n", " s = s.replace(\" \", \"_\")\n", " return f\" [{s}] \"\n", "\n", "def _partial_word_repl(m):\n", " s = m.group()\n", " word = s.strip(\".-\")\n", " probably_fragment = word not in COMMON_EN_WORDS\n", " if probably_fragment:\n", " return f\"{word}* -- \"\n", " else:\n", " return f\"{word} -- \"\n", "\n", "ALLOWED_TAGS = set([\n", " \"laughter\", \"foreign_language\", \"inaudible\", \"music\", \"redacted\"\n", "])\n", "\n", "def _standardize_tags(text):\n", " # collapse meta tags and filter\n", " for m in list(re.finditer(r\"\\[.*?\\]\", text))[::-1]:\n", " tag_text = text[m.start() + 1:m.end() - 1]\n", " # collapse notation\n", " if \"foreign\" in tag_text:\n", " tag_text = \"foreign_language\"\n", " if \"laugh\" in tag_text:\n", " tag_text = \"laughter\"\n", " if \"indistinct\" in tag_text:\n", " tag_text = \"inaudible\"\n", " if \"music\" in tag_text or \"\\\"\" in tag_text or \"plays\" in tag_text or \"sing\" in tag_text:\n", " tag_text = \"music\"\n", " # filter\n", " if tag_text in ALLOWED_TAGS:\n", " cleaned_tag_text = f\"[{tag_text}]\"\n", " else:\n", " cleaned_tag_text = \"\"\n", " text = text[:m.start()] + cleaned_tag_text + text[m.end():]\n", " return text\n", " \n", "def parse_rev_captions(transcript_srt, lang=\"en\"):\n", " if lang != \"en\":\n", " raise NotImplementedError(\"only en is supported for now.\")\n", " cur_speaker_id = 0\n", " token_list = []\n", " for block in _parse_srt(transcript_srt):\n", " lines = block[\"text_lines\"]\n", " if len(lines) == 0:\n", " continue\n", " # handle music lyrics\n", " annotated_lines = []\n", " cur_lyrics = False\n", " for line in lines:\n", " is_start = re.match(r\"^\\s*♪.*\", line)\n", " is_end = re.match(r\".*♪\\s*$\", line)\n", " if is_start:\n", " line = line.strip()[1:]\n", " if is_end:\n", " line = line.strip()[:-1]\n", " if is_start and is_end:\n", " annotated_lines.append((line, True))\n", " cur_lyrics = False\n", " elif is_start:\n", " annotated_lines.append((line, True))\n", " cur_lyrics = True\n", " elif is_end:\n", " annotated_lines.append((line, True))\n", " cur_lyrics = False\n", " else:\n", " annotated_lines.append((line, False))\n", " if re.match(r\".*♪\\s*$\", line):\n", " cur_lyrics = False\n", " # add actual lines\n", " for raw_line, is_lyrics in annotated_lines:\n", " # handle speaker change\n", " line = raw_line.strip()\n", " if line[:1] == \"-\":\n", " cur_speaker_id += 1\n", " line = line[1:].strip()\n", " # handle meta tags\n", " if re.search(r\"\\([^\\)]*\\(\", line): # treat nested by brute forcing\n", " line = re.sub(r\"\\((.*)\\)\", _tag_repl, line)\n", " line = re.sub(r\"\\((.*?)\\)\", _tag_repl, line)\n", " # handle redacted\n", " line = re.sub(r\"[^\\s]*\\*+\", \" [redacted] \", line)\n", " # handle hesitations and word fragments\n", " line = re.sub(r\"[A-Za-z](\\.{2,}|\\-(\\s|$))\", _partial_word_repl, line)\n", " line = re.sub(r\"\\.{2,}\", \" \", line) # we only keep interruptions as --\n", " # collapse meta tags and discard if necessary\n", " line = _standardize_tags(line)\n", " # add tokens\n", " line = normalize_whitespace(line)\n", " if len(line) == 0:\n", " continue\n", " tokens = []\n", " for s in line.split():\n", " # identify word fragment\n", " if re.match(r\"^[^\\*]+\\*$\", s):\n", " is_fragment = True\n", " s = s[:-1]\n", " else:\n", " is_fragment = False\n", " if s[:1] == \"[\" and s[-1:] == \"]\":\n", " is_text = False\n", " elif s == \"--\":\n", " is_text = False\n", " else:\n", " is_text = True\n", " if is_text:\n", " token = Token(\n", " s, \n", " speaker_id=cur_speaker_id, \n", " type=TEXT, \n", " metadata={\"is_lyrics\": is_lyrics},\n", " )\n", " else:\n", " token = Token(s, type=TAG)\n", " if token.type == TAG and len(tokens) > 0 and token == tokens[-1]:\n", " # skip consecutive meta tokens\n", " continue\n", " # TODO: deal with things like \"done [music] .\" can we use other rev parser?\n", " tokens.append(token)\n", " if is_fragment:\n", " tokens.append(Token(\"[word_fragment_boundary]\", type=TAG))\n", " token_list.extend(tokens)\n", " \n", " transcript_tokens = Tokens(token_list)\n", " return transcript_tokens" ] }, { "cell_type": "code", "execution_count": 5, "id": "00e04fcf", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 6, "id": "936b45dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "100/100 segments retained in 6be4d523-fac5-4778-b7bd-b05dc31ea43b.\n", "100/100 segments retained in f8446d0f-0b19-4da5-b9af-60d3da3abd2c.\n", "100/100 segments retained in 110eb1a6-1005-4aa9-9b68-e69456e6076f.\n", "100/100 segments retained in 7ea0c6bb-c71b-4835-8f9b-d70c87128c42.\n", "100/100 segments retained in acc78c33-25fa-4d16-9e02-6c4a74f5b373.\n", "100/100 segments retained in 1bebc5b6-3325-4215-a678-a0ef25b3f553.\n", "44/44 segments retained in e5565235-276f-4233-8039-ba90aaef5d93.\n" ] } ], "source": [ "# from suno_utils.web.rev import parse_transcript\n", "\n", "segment_transcripts = []\n", "for m in audio_rev_metas:\n", " srt_filepath = os.path.join(FROM_REV_DIR, m[\"uid\"] + \".srt\")\n", " if not os.path.exists(srt_filepath):\n", " print(\"not found, skipping...\")\n", " continue\n", " with open(srt_filepath) as f:\n", " transcript_srt = f.read()\n", " rev_transcript = parse_rev_captions(transcript_srt)\n", " n_segments = int(len(m[\"segments_meta\"]) / 2)\n", " token_segments = get_segments(rev_transcript, n_segments, global_indicator_speaker=False)\n", " segment_transcripts_chunk = []\n", " missing_ids = set(range(n_segments))\n", " for segment_idx, tokens in token_segments:\n", " segment_info = m[\"segments_meta\"][segment_idx * 2 + 1]\n", " segment_transcripts_chunk.append({\n", " \"uid\": segment_info[\"orginial_audio_uid\"],\n", " \"duration_s\": segment_info[\"duration_s\"],\n", " \"transcript\": tokens.anonymize_speakers(),\n", " })\n", " missing_ids -= set([segment_idx])\n", " missing_ids = sorted([str(n) for n in missing_ids])\n", " missing_str = \"\"\n", " if len(missing_ids) > 0:\n", " missing_str = \" (\" + \",\".join(missing_ids[:3])\n", " if len(missing_ids) > 3:\n", " missing_str += \"...\"\n", " missing_str += \")\"\n", " print(\"{}/{} segments retained in {}{}.\".format(\n", " len(segment_transcripts_chunk), n_segments, m[\"uid\"], missing_str\n", " ))\n", " segment_transcripts.extend(segment_transcripts_chunk)" ] }, { "cell_type": "code", "execution_count": 9, "id": "2cbe6494", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "02740476-28bf-433f-8e98-9c739efcb476\n", "Bob the Builder.\n", "----------\n", "1034ef38-79aa-48de-a908-0dd6a65c1a36\n", "I can't help.\n", "----------\n", "2c14f957-2e06-46fa-b1b1-b7d038f31b0c\n", "I want my mommy.\n", "----------\n", "afb73ee5-ddd2-4d29-8136-8387f42a5f4c\n", "Get out of here.\n", "----------\n", "7c0189ac-ea22-4efc-b742-ab2279c4fdcc\n", "I'm only high glitching to warm up.\n", "----------\n" ] } ], "source": [ "for m in segment_transcripts[:5]:\n", " print(m[\"uid\"])\n", " print(m[\"transcript\"].text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 10, "id": "90178e26", "metadata": {}, "outputs": [], "source": [ "# save as json\n", "import copy\n", "l = []\n", "for m in segment_transcripts:\n", " e = copy.deepcopy(m)\n", " e[\"transcript\"] = e[\"transcript\"].as_dict()\n", " l.append(e)\n", "with open(os.path.join(BASE_DIR, \"parsed_transcripts.json\"), \"w\") as f:\n", " json.dump(l, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "fdd8a85f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2cb9351c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c34c0db7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9937474e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2624f95e", "metadata": {}, "source": [ "## prepare pipeline" ] }, { "cell_type": "code", "execution_count": 11, "id": "9433c48a", "metadata": {}, "outputs": [], "source": [ "# map segment id to rev submission id\n", "with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", " audio_rev_metas = json.load(f)\n", "segment_to_rev_id = {}\n", "for m in audio_rev_metas:\n", " rev_id = m[\"uid\"]\n", " for mm in m[\"segments_meta\"]:\n", " if mm[\"type\"] != \"speech\":\n", " continue\n", " segment_to_rev_id[mm[\"orginial_audio_uid\"]] = rev_id" ] }, { "cell_type": "code", "execution_count": 12, "id": "a29859c9", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(BASE_DIR, \"parsed_transcripts.json\")) as f:\n", " segment_transcripts = json.load(f)" ] }, { "cell_type": "code", "execution_count": 13, "id": "020e71bb", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 644/644 [00:03<00:00, 199.66it/s]\n" ] } ], "source": [ "manifest = []\n", "for m in tqdm.tqdm(segment_transcripts):\n", " segment_filepath = os.path.join(AUDIO_DIR, m[\"uid\"] + \".mp3\") \n", " new_segment_filepath = os.path.join(SEGMENTS_DIR, \"{}.wav\".format(m[\"uid\"]))\n", " if not os.path.exists(new_segment_filepath):\n", " audio = Audio.from_file(segment_filepath, sample_rate=16_000, byte_width=2)\n", " audio.to_wav(new_segment_filepath)\n", " manifest.append({\n", " \"id\": m[\"uid\"],\n", " \"uri\": new_segment_filepath,\n", " \"duration_s\": m[\"duration_s\"],\n", " \"transcript\": {\"tokens\": m[\"transcript\"]},\n", " \"ignore_segment\": False,\n", " \"ignore_reason\": \"\",\n", " \"source_audio_offset_s\": 0,\n", " \"source_audio_uri\": segment_filepath,\n", " \"source_audio_id\": m[\"uid\"],\n", " \"source_transcript_id\": segment_to_rev_id[m[\"uid\"]],\n", " \"source_transcript_uri\": os.path.join(FROM_REV_DIR, segment_to_rev_id[m[\"uid\"]] + \".srt\"),\n", " })" ] }, { "cell_type": "code", "execution_count": 14, "id": "cee8873a", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\"), \"w\") as f:\n", " json.dump(manifest, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "3810b78a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6cf2b71d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "651e980e", "metadata": {}, "source": [ "## QA" ] }, { "cell_type": "code", "execution_count": null, "id": "a8dae9e1", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6a5f0e8c", "metadata": {}, "outputs": [], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = segments_manifest[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "33e34ad9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2c7eff44", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "64fe98cc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7b8ddb47", "metadata": {}, "source": [ "## run pipeline" ] }, { "cell_type": "code", "execution_count": 15, "id": "1e319351", "metadata": {}, "outputs": [], "source": [ "# # reset pipeline\n", "# !mv /mnt/data-ssd-1/data/private/customer/speechly/2022-08-30_gymclass_1h/pipeline/2022_08_30/artifacts/01_segment_meta.json /tmp/01_segment_meta.json\n", "# !rm -rf /mnt/data-ssd-1/data/private/customer/speechly/2022-08-30_gymclass_1h/pipeline/2022_08_30/artifacts/*\n", "# !mv /tmp/01_segment_meta.json /mnt/data-ssd-1/data/private/customer/speechly/2022-08-30_gymclass_1h/pipeline/2022_08_30/artifacts/" ] }, { "cell_type": "code", "execution_count": null, "id": "7fcf3139", "metadata": {}, "outputs": [], "source": [ "# CUDA_VISIBLE_DEVICES=0 python /home/georg/notebooks/customers/speechly/gymclass_sample/run_modified_pipeline.py" ] }, { "cell_type": "code", "execution_count": null, "id": "d6d3461c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fb1aec05", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "cfa1946d", "metadata": {}, "source": [ "## look at results" ] }, { "cell_type": "code", "execution_count": 16, "id": "30275498", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1.0 hours\n" ] } ], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"02_segment_meta_post_norm_and_asr.json\")) as f:\n", " segments_manifest = json.load(f)\n", "print(round(np.sum([e[\"duration_s\"] for e in segments_manifest]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "5848265d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "b2d89820-acc9-4bde-a182-e947c411ffd3\n", "Ah, meow!\n", "ah meow\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "7ec66b2b-3892-4f2e-83cb-c08ab9634abb\n", "Of course! Biggest air ball of 2022! Bro, am I right, or am I wrong?\n", "of course biggest air ball of twenty twenty two bro am i right or am i wrong\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "67a57adf-5ec9-4b00-a0be-86ef6eff76bb\n", "Getting progress? What? I just joined the game.\n", "getting progress what i just joined the game\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "d9987f2e-4c5d-4e77-86f5-b4be5906763b\n", "No, I'm not joining.\n", "no i'm not joining\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n", "fc452a10-6b0b-4d26-b9dc-7265b7cef5de\n", "Oh, no. Wait.\n", "oh no wait\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = segments_manifest[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"id\"])\n", " print(Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " print(Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n", " Audio.from_file(e[\"uri\"]).play()\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 19, "id": "34165d5a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "full: Who asked?\n", "norm: who asked\n", "asr : who asked\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "uid = \"b038b066-0d52-4d27-b1fb-a6643c825eee\"\n", "for e in segments_manifest:\n", " if e[\"id\"] == uid:\n", " print(\"full:\", Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text)\n", " print(\"norm:\", Tokens.from_dict(e[\"transcript_normalized\"][\"tokens\"]).text)\n", " print(\"asr :\", e[\"transcript_asr\"][\"transcript\"])\n", " Audio.from_file(e[\"uri\"]).play()" ] }, { "cell_type": "code", "execution_count": 28, "id": "afd18433", "metadata": {}, "outputs": [], "source": [ "# l = []\n", "# for e in segments_manifest:\n", "# l.extend(re.findall(r\"[a-zA-Z]+\\-[a-zA-Z]+\", Tokens.from_dict(e[\"transcript\"][\"tokens\"]).text))" ] }, { "cell_type": "code", "execution_count": null, "id": "ab4c510b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6b99850b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dc0d9e56", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "938c2793", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6b0192cb", "metadata": {}, "source": [ "## format output for speechly styleguide" ] }, { "cell_type": "code", "execution_count": 20, "id": "336f6909", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2022-09-01 13:13:11-- https://www.cs.cmu.edu/~biglou/resources/bad-words.txt\n", "Resolving www.cs.cmu.edu (www.cs.cmu.edu)... 128.2.42.95\n", "Connecting to www.cs.cmu.edu (www.cs.cmu.edu)|128.2.42.95|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 11169 (11K) [text/plain]\n", "Saving to: ‘bad-words.txt’\n", "\n", "bad-words.txt 100%[===================>] 10.91K --.-KB/s in 0.001s \n", "\n", "2022-09-01 13:13:12 (7.37 MB/s) - ‘bad-words.txt’ saved [11169/11169]\n", "\n" ] } ], "source": [ "!wget https://www.cs.cmu.edu/~biglou/resources/bad-words.txt" ] }, { "cell_type": "code", "execution_count": 205, "id": "798221f7", "metadata": {}, "outputs": [], "source": [ "# TODO: do we need to keep dashes during norm? what about M-mhm uh-huh, ...\n", "# TODO:\n", "# https://www.cs.cmu.edu/~biglou/resources/bad-words.txt\n", "# https://github.com/zacanger/profane-words" ] }, { "cell_type": "code", "execution_count": 47, "id": "7c3f2643", "metadata": {}, "outputs": [], "source": [ "with open(\"bad-words.txt\") as f:\n", " profane_words = set([s_clean for s in f.read().strip().split(\"\\n\") if len((s_clean := s.strip())) > 0])\n", "profane_words_ptn = \"|\".join(re.escape(s.lower()) for s in sorted(profane_words, key=len, reverse=True))\n", "PROFANE_WORDS_RE = re.compile(r\"\\b(?:\" + profane_words_ptn + r\")\\b\")\n", "\n", "def _contains_profanity(text):\n", " return bool(PROFANE_WORDS_RE.search(text.lower()))" ] }, { "cell_type": "code", "execution_count": 74, "id": "290b5f06", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.harvest import get_file_name\n", "\n", "with open(os.path.join(BASE_DIR, \"raw_data_manifest.json\")) as f:\n", " filepaths_info = json.load(f)\n", "id_to_original_id_map = {e[0]: get_file_name(e[2], keep_extension=True) for e in filepaths_info}\n", "inv_id_to_original_id_map = {v: k for k, v in id_to_original_id_map.items()}\n", "\n", "with open(os.path.join(ARTIFACTS_DIR, \"02_segment_meta_post_norm_and_asr.json\")) as f:\n", " segments_manifest = json.load(f)" ] }, { "cell_type": "code", "execution_count": 75, "id": "ecd8a4d5", "metadata": {}, "outputs": [], "source": [ "def _convert_full_transcript(tokens):\n", " transcript_l = []\n", " for turn in tokens.speaker_turns:\n", " turn_tokens = Tokens.from_dict(turn[\"tokens\"])\n", " active_lyrics = False\n", " text_l = []\n", " for t in turn_tokens:\n", " is_lyrics = False\n", " if t.metadata is not None:\n", " is_lyrics = t.metadata[\"is_lyrics\"]\n", " if is_lyrics and not active_lyrics:\n", " text_l.append(\"\")\n", " if not is_lyrics and active_lyrics:\n", " text_l.append(\"\")\n", " text_l.append(t.value)\n", " active_lyrics = is_lyrics\n", " if active_lyrics:\n", " text_l.append(\"\")\n", " text = \" \".join(text_l)\n", " text = re.sub(r\"\\\\s+\", \"\", text)\n", " text = re.sub(r\"\\s+\\<\\/lyrics\\>\", \"\", text)\n", " transcript_l.append(text)\n", " transcript = \" [speaker_change] \".join(transcript_l)\n", " transcript = re.sub(r\"\\s+\\[word\\_fragment\\_boundary\\]\", \"*\", transcript)\n", " # fix some basic parse mistackes\n", " transcript = re.sub(r\"^\\s*\\[music\\] \\[speaker\\_change\\]\\s*\", \"[music] \", transcript)\n", " transcript = normalize_whitespace(transcript)\n", " return transcript\n", "\n", "def _convert_norm_transcript(tokens):\n", " transcript = tokens.text\n", " transcript = transcript.replace(\" [word_fragment_boundary]\", \"*\")\n", " transcript = transcript.replace(\"[foreign_language]\", \"\")\n", " transcript = transcript.replace(\"[inaudible]\", \"\")\n", " transcript = transcript.replace(\"--\", \" \")\n", " transcript = re.sub(r\"\\[.*?\\]\", \" \", transcript)\n", " transcript = normalize_whitespace(transcript)\n", " return transcript" ] }, { "cell_type": "code", "execution_count": 90, "id": "b2465b35", "metadata": {}, "outputs": [], "source": [ "# jsonl \n", "\n", "# {\n", "# \"audio\": \"0a4ac9ac-42a4-4e77-91ad-862b0509d913.wav.\", // the audio file name\n", "# \"transcript\": \".....\", // The transcript of the audio\n", "# \"tags\": [ ] // The \"OFF\" and/or \"AC\" tags if relevant to the utterance\n", "# }\n", "# Audio corrupted utterance (AC): for only severely corrupted audio files\n", "# Offensive utterance (OFF): profanities, bullying language, hate speech included in the audio file\n", "output_metadata = []\n", "for n, m in enumerate(segments_manifest):\n", " norm_tokens = Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"])\n", " tokens = Tokens.from_dict(m[\"transcript\"][\"tokens\"])\n", " norm_transcript = _convert_norm_transcript(norm_tokens)\n", " transcript = _convert_full_transcript(tokens)\n", " tags = []\n", " if _contains_profanity(norm_transcript):\n", " tags.append(\"OFF\")\n", " original_filename = id_to_original_id_map[m[\"id\"]]\n", " \n", "# if \"foreign\" in transcript:\n", "# print(norm_transcript)\n", "# print(\"-\")\n", "# print(transcript)\n", "# print(\"-\"*10)\n", "# if len(norm_transcript) == 0:\n", "# print(norm_transcript)\n", "# print(\"-\")\n", "# print(transcript)\n", "# print(\"-\"*10)\n", " \n", " # some manual overrides\n", " # TODO: use CER here in future to punt and maybe reannotate\n", " if original_filename == \"c7bfc752-be04-4c91-b018-bf809ad19323.mp3\":\n", " transcript = transcript.replace(\"Banja not to this barely\", \"[foreign_language] Chef Bally\")\n", " norm_transcript = norm_transcript.replace(\"we ' re\", \"we're\")\n", " if original_filename == \"cc6284ac-dc8b-420c-9bd0-8470d2e2d88d.mp3\":\n", " transcript = \"Damn.\"\n", " norm_transcript = \"damn\"\n", " tags = [\"OFF\"]\n", " if original_filename == \"b451b2ff-0dd2-4a8b-a317-34ea67f172bd.mp3\":\n", " transcript = transcript.replace(\"I can't\", \"I can't, I can't\")\n", " norm_transcript = norm_transcript.replace(\"i can't\", \"i can't i can't\")\n", " if original_filename == \"4600e16e-fd7d-4c69-9e1d-49b1a3c2b088.mp3\":\n", " transcript = transcript.replace(\"y'all\", \"yo\")\n", " norm_transcript = norm_transcript.replace(\"y'all\", \"yo\") \n", " if original_filename == \"3cd8a90f-8475-4d7c-84f3-07e9b7aa0227.mp3\":\n", " transcript = \"What?\"\n", " norm_transcript = \"what\"\n", " if original_filename == \"558a3f70-8f55-4b7e-9b14-6d6e2a436218.mp3\":\n", " transcript = \"\"\n", " norm_transcript = \"[inaudible]\"\n", " if original_filename == \"1034ef38-79aa-48de-a908-0dd6a65c1a36.mp3\":\n", " transcript = \"I can't jump.\"\n", " norm_transcript = \"i can't jump\"\n", " output_metadata.append({\n", " \"audio\": original_filename,\n", " \"transcript\": normalize_whitespace(norm_transcript),\n", " \"transcript_full\": normalize_whitespace(transcript),\n", " \"tags\": tags,\n", " })" ] }, { "cell_type": "code", "execution_count": 91, "id": "00cecd18", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"delivery_meta.jsonl\"), \"w\") as f:\n", " for e in output_metadata:\n", " f.write(json.dumps(e))\n", " f.write(\"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "id": "4fd2b759", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fc59e298", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "635386fe", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6b0b614f", "metadata": {}, "source": [ "## Final QA" ] }, { "cell_type": "code", "execution_count": 85, "id": "a61aa689", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(ARTIFACTS_DIR, \"delivery_meta.jsonl\")) as f:\n", " output_metadata = [json.loads(e) for e in f.read().strip().split(\"\\n\")]" ] }, { "cell_type": "code", "execution_count": 93, "id": "8a5d9f26", "metadata": {}, "outputs": [], "source": [ "# for e in output_metadata:\n", "# if e[\"audio\"] == \"1034ef38-79aa-48de-a908-0dd6a65c1a36.mp3\":\n", "# print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", "# Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", "# print(e[\"transcript\"])\n", "# print(\"-\")\n", "# print(e[\"transcript_full\"])\n", "# break" ] }, { "cell_type": "code", "execution_count": 235, "id": "201ac267", "metadata": {}, "outputs": [], "source": [ "# # figure out rev\n", "# with open(os.path.join(TO_REV_DIR, f\"audio_rev_metas.json\")) as f:\n", "# audio_rev_metas = json.load(f)\n", "# uid = inv_id_to_original_id_map[\"harrisnt_-7072462733219728682.m4v\"]\n", "# for rev_m in audio_rev_metas:\n", "# for seg_m in rev_m[\"segments_meta\"]:\n", "# if seg_m.get(\"orginial_audio_uid\") == uid:\n", "# print(seg_m)" ] }, { "cell_type": "code", "execution_count": 86, "id": "9ccfd86b", "metadata": {}, "outputs": [], "source": [ "# import random\n", "# # random ones\n", "# # l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "# l = [e for e in output_metadata if len(e[\"transcript\"]) == 0]\n", "# random.shuffle(l)\n", "# for e in l[:5]:\n", "# print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", "# Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", "# print(e[\"transcript\"])\n", "# print(\"-\")\n", "# print(e[\"transcript_full\"])\n", "# print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": 87, "id": "a3f6a7dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "20b330a1-bac1-46b0-b1cc-390bc037135a.mp3 - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "yo\n", "-\n", "Yo.\n", "----------\n", "39f2b97b-1fd7-430d-9276-0a935d838d8b.mp3 - profanity: True\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "fuck yeah i got this little bitch here boy ill kiss your mother in the mouth fuck\n", "-\n", "Fuck! Yeah, I got this little bitch here. Boy, Ill kiss your mother in the mouth. Fuck.\n", "----------\n", "4f7054d8-1899-4612-8f16-4c88dfd24a61.mp3 - profanity: True\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "i wanna pee now\n", "-\n", "I wanna pee now.\n", "----------\n", "2394b9c8-5c5a-4379-83b2-a2ec94d96888.mp3 - profanity: True\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "well dude i get that you snatch it once but you don't have to keep snatching me like\n", "-\n", "Well, dude, I get that you snatch it once but you don't have to keep snatching me like.\n", "----------\n", "429014a4-25c5-4b83-8b07-d3a91f3c3e38.mp3 - profanity: False\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "i am i am\n", "-\n", "I am. I am.\n", "----------\n" ] } ], "source": [ "import random\n", "# random ones\n", "# l = [e for e in segments_manifest if len(e[\"transcript\"][\"tokens\"]) == 0]\n", "l = output_metadata[:]\n", "random.shuffle(l)\n", "for e in l[:5]:\n", " print(e[\"audio\"], \"- profanity:\", \"OFF\" in e[\"tags\"])\n", " Audio.from_file(os.path.join(BASE_DIR, \"audio\", e[\"audio\"])).play()\n", " print(e[\"transcript\"])\n", " print(\"-\")\n", " print(e[\"transcript_full\"])\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "cca2d891", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9cf9604f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d51f20f8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "36ae3093", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 3, "id": "18650ce8", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-08-22 20:16:24 optimizers:77] Could not import distributed_fused_adam optimizer from Apex\n" ] } ], "source": [ "import tqdm\n", "import time\n", "from suno_utils.tasks.asr import transcribe\n", "from suno_utils.audio import Tokens\n", "\n", "# with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\")) as f:\n", "with open(os.path.join(ARTIFACTS_DIR, \"03_segment_meta_post_timestamp.json\")) as f:\n", " segments_manifest = json.load(f)" ] }, { "cell_type": "markdown", "id": "a063a729", "metadata": {}, "source": [ "### transcript parsing" ] }, { "cell_type": "code", "execution_count": 92, "id": "ce0a0348", "metadata": {}, "outputs": [], "source": [ "import json\n", "import logging\n", "import os\n", "import re\n", "import shutil\n", "from statistics import mode\n", "import uuid\n", "\n", "from Bio import pairwise2\n", "import tqdm\n", "\n", "from suno_utils.audio import Audio, Tokens\n", "from suno_utils.datasets.helpers import get_basename_with_segments_dir\n", "from suno_utils.datasets.rev_podcast.pipeline import step_1, step_2\n", "from suno_utils.datasets.rev_podcast.process_data import validate\n", "from suno_utils.web.rev import parse_transcript\n", "\n", "\n", "DIGITS = list(\"0123456789\")\n", "DIGITS_STRINGS = \"zero one two three four five six seven eight nine\".split()\n", "\n", "\n", "logger = logging.getLogger(__name__)\n", "\n", "\n", "def _is_numeric(text):\n", " text_bits = re.sub(r\"[^\\sa-z0-9]\", \"\", text.lower()).strip().split()\n", " return all([s in set(DIGITS_STRINGS) or re.match(r\"[0-9]+\", s) for s in text_bits])\n", "\n", "\n", "def _digitify(text):\n", " text = text.lower()\n", " for digit, digit_str in zip(DIGITS, DIGITS_STRINGS):\n", " text = re.sub(r\"\\b\" + digit_str + r\"\\b\", f\" {digit} \", text)\n", " text = re.sub(r\"[^0-9]\", \"\", text)\n", " return text\n", "\n", "\n", "def _find_valid_indicators(test_list, known_list):\n", " if \"*\" in test_list or \"*\" in known_list:\n", " raise ValueError(\"`*` symbol is reserved,\")\n", " out = pairwise2.align.globalxx(test_list, known_list, gap_char=[\"*\"])\n", " return [a == b for a, b in zip(out[0].seqA, out[0].seqB)]\n", "\n", "\n", "def get_segments_new(rev_transcript, n_segments, global_indicator_speaker=True):\n", " if global_indicator_speaker:\n", " # find indicator speaker\n", " indicator_segments = [\n", " t[\"speaker_id\"]\n", " for t in rev_transcript.speaker_turns\n", " if _is_numeric(t[\"plaintext\"])\n", " ]\n", " if len(indicator_segments) == 0:\n", " logger.warning(\"no indicators found\")\n", " return []\n", " indicator_speaker_id = mode(indicator_segments)\n", " else:\n", " indicator_speaker_id = None\n", " # parse numbers for indicator speaker\n", " indicator_tokens = []\n", " prev_n = -2\n", " last_speaker_id = None\n", " for n, token in enumerate(rev_transcript.tokens):\n", " # if we know indicator speaker id then skip others\n", " if indicator_speaker_id is not None and indicator_speaker_id != token.speaker_id:\n", " last_speaker_id = token.speaker_id\n", " continue\n", " parsed_number = _digitify(token.value)\n", " if len(parsed_number) == 0:\n", " last_speaker_id = token.speaker_id\n", " continue\n", " # if multiple tokens in a row pass then add to previous\n", " if n - prev_n == 1 and last_speaker_id == token.speaker_id:\n", " indicator_tokens[-1] = (\n", " indicator_tokens[-1][0],\n", " indicator_tokens[-1][1] + parsed_number,\n", " )\n", " elif last_speaker_id != token.speaker_id:\n", " indicator_tokens.append((n, parsed_number))\n", " else:\n", " last_speaker_id = token.speaker_id\n", " continue\n", " prev_n = n\n", " last_speaker_id = token.speaker_id\n", "\n", " is_indicators_valid = _find_valid_indicators(\n", " [s for _, s in indicator_tokens],\n", " [str(n).zfill(2) for n in range(n_segments)]\n", " )\n", " correct_indicators = [\n", " (int(s), idx) for (idx, s), is_valid in zip(indicator_tokens, is_indicators_valid)\n", " if is_valid\n", " ]\n", "\n", " # fix missing indicators if possible\n", " missing_indicators = list(set(range(n_segments)) - set([n for n, _ in correct_indicators]))\n", " added_indicators = []\n", " for missing_indicator in missing_indicators:\n", " # obtain search index\n", " l = [e for e in correct_indicators if e[0] < missing_indicator]\n", " if len(l) > 0:\n", " start_idx = l[-1][-1]\n", " else:\n", " start_idx = 0\n", " l = [e for e in correct_indicators if e[0] > missing_indicator]\n", " if len(l) > 0:\n", " end_idx = l[0][-1]\n", " else:\n", " end_idx = len(rev_transcript.tokens)\n", " # find index within\n", " missing_indicator_str = str(missing_indicator).zfill(2)\n", " indicator_token_idx = None\n", " n_matched = 0\n", " for n, t in enumerate(rev_transcript.tokens[start_idx:end_idx]):\n", " if _digitify(t.value) == missing_indicator_str[n_matched]:\n", " n_matched += 1\n", " elif len(_digitify(t.value)) > 0 and int(_digitify(t.value)) == missing_indicator:\n", " n_matched = len(missing_indicator_str)\n", " else:\n", " n_matched = 0\n", " if n_matched == len(missing_indicator_str):\n", " indicator_token_idx = start_idx + n + 1 - len(missing_indicator_str)\n", " break\n", " if indicator_token_idx is not None:\n", " added_indicators.append((int(missing_indicator_str), indicator_token_idx))\n", " correct_indicators = sorted(correct_indicators + added_indicators, key=lambda x: x[0])\n", "\n", " # get relevant tokens for all consecutive ones\n", " token_segments = []\n", " for (indicator_index_l, token_index_l), (indicator_index_r, token_index_r) in zip(\n", " correct_indicators[:-1], correct_indicators[1:]\n", " ):\n", " if indicator_index_l == indicator_index_r - 1:\n", " relevant_tokens = [\n", " t\n", " for t in rev_transcript.tokens[token_index_l:token_index_r]\n", " if indicator_speaker_id is None or indicator_speaker_id != t.speaker_id\n", " ]\n", " token_segments.append((indicator_index_l, Tokens(relevant_tokens)))\n", " # check last one\n", " indicator_index_l, token_index_l = correct_indicators[-1]\n", " if indicator_index_l == n_segments - 1:\n", " relevant_tokens = [\n", " t\n", " for t in rev_transcript.tokens[token_index_l:]\n", " if indicator_speaker_id is None or indicator_speaker_id != t.speaker_id\n", " ]\n", " token_segments.append((indicator_index_l, Tokens(relevant_tokens)))\n", " # if we don't have a global indicator we want to remove the first speaker from each segment\n", " if indicator_speaker_id is None:\n", " cleaned_token_segments = []\n", " for indicator_idx, tokens in token_segments:\n", " if len(tokens) == 0:\n", " raise ValueError(\"no tokens found in indicator segment, this should not happen.\")\n", " cleaned_tokens = []\n", " for n, t in enumerate(tokens[1:]):\n", " if t.speaker_id != tokens[0].speaker_id or len(_digitify(t.value)) == 0:\n", " cleaned_tokens = tokens[n+1:]\n", " break\n", " cleaned_token_segments.append((indicator_idx, Tokens(cleaned_tokens)))\n", " else:\n", " cleaned_token_segments = token_segments\n", " return cleaned_token_segments" ] }, { "cell_type": "code", "execution_count": 93, "id": "f93ca593", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "84/100 segments retained in f8446d0f-0b19-4da5-b9af-60d3da3abd2c (55,56,57...).\n" ] } ], "source": [ "for m in audio_rev_metas:\n", " if m[\"uid\"] != \"f8446d0f-0b19-4da5-b9af-60d3da3abd2c\":\n", " continue\n", " srt_filepath = os.path.join(FROM_REV_DIR, m[\"uid\"] + \".srt\")\n", " with open(srt_filepath) as f:\n", " transcript_srt = f.read()\n", " rev_transcript = parse_rev_captions(transcript_srt)\n", " n_segments = int(len(m[\"segments_meta\"]) / 2)\n", " \n", " \n", " # TODO: need to fix 'get_segments' here.\n", " token_segments = get_segments_new(rev_transcript, n_segments, global_indicator_speaker=False)\n", " \n", " \n", " segment_transcripts_chunk = []\n", " missing_ids = set(range(n_segments))\n", " for segment_idx, tokens in token_segments:\n", " segment_info = m[\"segments_meta\"][segment_idx * 2 + 1]\n", " segment_transcripts_chunk.append({\n", " \"uid\": segment_info[\"orginial_audio_uid\"],\n", " \"duration_s\": segment_info[\"duration_s\"],\n", " \"transcript\": tokens.anonymize_speakers(),\n", " })\n", " missing_ids -= set([segment_idx])\n", " missing_ids = sorted([str(n) for n in missing_ids])\n", " missing_str = \"\"\n", " if len(missing_ids) > 0:\n", " missing_str = \" (\" + \",\".join(missing_ids[:3])\n", " if len(missing_ids) > 3:\n", " missing_str += \"...\"\n", " missing_str += \")\"\n", " print(\"{}/{} segments retained in {}{}.\".format(\n", " len(segment_transcripts_chunk), n_segments, m[\"uid\"], missing_str\n", " ))\n", " break" ] }, { "cell_type": "code", "execution_count": null, "id": "f1684092", "metadata": {}, "outputs": [], "source": [ "(31, Tokens(text=`3-1. Who`)),\n", "(32, Tokens(text=`3-2. You l...`))," ] }, { "cell_type": "code", "execution_count": 124, "id": "b4cf48f8", "metadata": {}, "outputs": [], "source": [ "global_indicator_speaker = False\n", "\n", "def _find_valid_indicators(test_list, known_list):\n", " if \"*\" in test_list or \"*\" in known_list:\n", " raise ValueError(\"`*` symbol is reserved,\")\n", " out = pairwise2.align.globalxx(test_list, known_list, gap_char=[\"*\"])\n", " is_valid_list = []\n", " for a, b in zip(out[0].seqA, out[0].seqB):\n", " if a != \"*\":\n", " is_valid_list.append(a == b)\n", " return is_valid_list" ] }, { "cell_type": "code", "execution_count": 125, "id": "1f4052c6", "metadata": {}, "outputs": [], "source": [ "indicator_speaker_id = None\n", "# parse numbers for indicator speaker\n", "indicator_tokens = []\n", "prev_n = -2\n", "last_speaker_id = None\n", "for n, token in enumerate(rev_transcript.tokens):\n", " parsed_number = _digitify(token.value)\n", " if len(parsed_number) == 0:\n", " last_speaker_id = token.speaker_id\n", " continue\n", " # if multiple tokens in a row pass then add to previous\n", " if n - prev_n == 1 and last_speaker_id == token.speaker_id:\n", " indicator_tokens[-1] = (\n", " indicator_tokens[-1][0],\n", " indicator_tokens[-1][1] + parsed_number,\n", " )\n", " elif last_speaker_id != token.speaker_id:\n", " indicator_tokens.append((n, parsed_number))\n", " else:\n", " last_speaker_id = token.speaker_id\n", " continue\n", " prev_n = n\n", " last_speaker_id = token.speaker_id\n", "\n", "is_indicators_valid = _find_valid_indicators(\n", " [s for _, s in indicator_tokens],\n", " [str(n).zfill(2) for n in range(n_segments)]\n", ")\n", "correct_indicators = [\n", " (int(s), idx) for (idx, s), is_valid in zip(indicator_tokens, is_indicators_valid)\n", " if is_valid\n", "]\n", "\n", "# fix missing indicators if possible\n", "missing_indicators = list(set(range(n_segments)) - set([n for n, _ in correct_indicators]))\n", "added_indicators = []\n", "for missing_indicator in missing_indicators:\n", " # obtain search index\n", " l = [e for e in correct_indicators if e[0] < missing_indicator]\n", " if len(l) > 0:\n", " start_idx = l[-1][-1]\n", " else:\n", " start_idx = 0\n", " l = [e for e in correct_indicators if e[0] > missing_indicator]\n", " if len(l) > 0:\n", " end_idx = l[0][-1]\n", " else:\n", " end_idx = len(rev_transcript.tokens)\n", " # find index within\n", " missing_indicator_str = str(missing_indicator).zfill(2)\n", " indicator_token_idx = None\n", " n_matched = 0\n", " for n, t in enumerate(rev_transcript.tokens[start_idx:end_idx]):\n", " if _digitify(t.value) == missing_indicator_str[n_matched]:\n", " n_matched += 1\n", " elif len(_digitify(t.value)) > 0 and int(_digitify(t.value)) == missing_indicator:\n", " n_matched = len(missing_indicator_str)\n", " else:\n", " n_matched = 0\n", " if n_matched == len(missing_indicator_str):\n", " indicator_token_idx = start_idx + n + 1 - len(missing_indicator_str)\n", " break\n", " if indicator_token_idx is not None:\n", " added_indicators.append((int(missing_indicator_str), indicator_token_idx))\n", "correct_indicators = sorted(correct_indicators + added_indicators, key=lambda x: x[0])\n", "\n", "# # get relevant tokens for all consecutive ones\n", "# token_segments = []\n", "# for (indicator_index_l, token_index_l), (indicator_index_r, token_index_r) in zip(\n", "# correct_indicators[:-1], correct_indicators[1:]\n", "# ):\n", "# if indicator_index_l == indicator_index_r - 1:\n", "# relevant_tokens = [\n", "# t\n", "# for t in rev_transcript.tokens[token_index_l:token_index_r]\n", "# ]\n", "# token_segments.append((indicator_index_l, Tokens(relevant_tokens)))\n", "# # check last one\n", "# indicator_index_l, token_index_l = correct_indicators[-1]\n", "# if indicator_index_l == n_segments - 1:\n", "# relevant_tokens = [\n", "# t\n", "# for t in rev_transcript.tokens[token_index_l:]\n", "# ]\n", "# token_segments.append((indicator_index_l, Tokens(relevant_tokens)))\n", "# # if we don't have a global indicator we want to remove the first speaker from each segment\n", "# cleaned_token_segments = []\n", "# for indicator_idx, tokens in token_segments:\n", "# if len(tokens) == 0:\n", "# raise ValueError(\"no tokens found in indicator segment, this should not happen.\")\n", "# cleaned_tokens = []\n", "# for n, t in enumerate(tokens[1:]):\n", "# if t.speaker_id != tokens[0].speaker_id or len(_digitify(t.value)) == 0:\n", "# cleaned_tokens = tokens[n+1:]\n", "# break\n", "# cleaned_token_segments.append((indicator_idx, Tokens(cleaned_tokens)))" ] }, { "cell_type": "code", "execution_count": 101, "id": "583a1d13", "metadata": {}, "outputs": [], "source": [ "# TODO: some tokens get merged: '2930', others missing '60', and some falsely recognized as valid '5657'" ] }, { "cell_type": "code", "execution_count": 121, "id": "8ebebf4a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 122, "id": "e7995e34", "metadata": {}, "outputs": [], "source": [ "is_indicators_valid = _find_valid_indicators2(\n", " [s for _, s in indicator_tokens],\n", " [str(n).zfill(2) for n in range(n_segments)]\n", ")\n", "correct_indicators2 = [\n", " (int(s), idx) for (idx, s), is_valid in zip(indicator_tokens, is_indicators_valid)\n", " if is_valid\n", "]" ] }, { "cell_type": "code", "execution_count": 123, "id": "6b627dc6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[(0, 0),\n", " (1, 2),\n", " (2, 7),\n", " (3, 8),\n", " (4, 13),\n", " (5, 16),\n", " (6, 42),\n", " (7, 46),\n", " (8, 52),\n", " (9, 59),\n", " (10, 73),\n", " (11, 75),\n", " (12, 81),\n", " (13, 92),\n", " (14, 94),\n", " (15, 133),\n", " (16, 153),\n", " (17, 157),\n", " (18, 161),\n", " (19, 165),\n", " (20, 179),\n", " (21, 191),\n", " (22, 196),\n", " (23, 198),\n", " (24, 204),\n", " (25, 214),\n", " (26, 219),\n", " (27, 221),\n", " (28, 228),\n", " (31, 235),\n", " (32, 238),\n", " (33, 242),\n", " (34, 246),\n", " (35, 250),\n", " (36, 254),\n", " (37, 256),\n", " (38, 261),\n", " (39, 268),\n", " (40, 270),\n", " (41, 275),\n", " (42, 287),\n", " (43, 293),\n", " (44, 297),\n", " (45, 301),\n", " (46, 305),\n", " (47, 314),\n", " (48, 323),\n", " (49, 328),\n", " (50, 339),\n", " (51, 347),\n", " (52, 353),\n", " (53, 358),\n", " (54, 366),\n", " (55, 396),\n", " (58, 405),\n", " (59, 409),\n", " (61, 423),\n", " (63, 431),\n", " (64, 442),\n", " (65, 453),\n", " (66, 460),\n", " (67, 463),\n", " (68, 466),\n", " (69, 469),\n", " (70, 471),\n", " (71, 481),\n", " (72, 483),\n", " (73, 493),\n", " (74, 497),\n", " (75, 500),\n", " (76, 509),\n", " (77, 511),\n", " (78, 515),\n", " (79, 518),\n", " (80, 527),\n", " (81, 530),\n", " (82, 533),\n", " (83, 541),\n", " (84, 548),\n", " (85, 551),\n", " (86, 560),\n", " (87, 563),\n", " (88, 579),\n", " (89, 586),\n", " (90, 590),\n", " (91, 595),\n", " (92, 600),\n", " (93, 616),\n", " (94, 622),\n", " (95, 632),\n", " (96, 634),\n", " (98, 639),\n", " (99, 642)]" ] }, "execution_count": 123, "metadata": {}, "output_type": "execute_result" } ], "source": [ "correct_indicators2" ] }, { "cell_type": "code", "execution_count": null, "id": "1b53988c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6a041156", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "10327061", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c4a4c7da", "metadata": {}, "source": [ "### mfa" ] }, { "cell_type": "code", "execution_count": 47, "id": "569c4d97", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.mfa import align_tokens" ] }, { "cell_type": "code", "execution_count": null, "id": "7c705fa4", "metadata": {}, "outputs": [], "source": [ "out = align_tokens(\n", " [m[\"uri\"] for m in segments_manifest],\n", " [Tokens.from_dict(m[\"transcript\"][\"tokens\"]) for m in segments_manifest],\n", " \"/home/georg/anaconda3/etc/profile.d/conda.sh\",\n", " \"mfa\",\n", " num_cores=40,\n", " dictionary_name=\"english_mfa\",\n", " acoustic_model_name=\"english_mfa\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "0c528aeb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "505458b2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f1783094", "metadata": {}, "source": [ "### norm" ] }, { "cell_type": "code", "execution_count": 86, "id": "d73061f9", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text_normalizer import (\n", " normalize, CUSTOM_NORM_RULES, _normalize, normalize_mp, NORMALIZER\n", ")" ] }, { "cell_type": "code", "execution_count": 8, "id": "d0d6662c", "metadata": {}, "outputs": [], "source": [ "# asr_predictions = transcribe([m[\"uri\"] for m in segments_manifest])" ] }, { "cell_type": "code", "execution_count": 198, "id": "ac2fd410", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "92\n" ] } ], "source": [ "for n, m in enumerate(segments_manifest):\n", " if m[\"id\"] == \"2ed3e928-fb78-44b1-89b1-572cd3301000\":\n", " print(n)\n", " break" ] }, { "cell_type": "code", "execution_count": 199, "id": "d59b4a2d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "Audio.from_file(segments_manifest[92][\"uri\"]).play()" ] }, { "cell_type": "code", "execution_count": 200, "id": "7d421c69", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"ok so as you guys know i'm getting married in a few weeks and i want to show you all some of my favorite wedding things that a fun so this is probably one my favorites it just came in this is my getting ready which is like this really sif really six really soft sok and these really fun wide whimsical tool sleeves with pearls tell me that's not the cutest and currently on my feet a adorable little slippers that i bought likere so freaking leading the look by hanging on this mrs ellis hanger also my wedding dress it has this little tool has our wedding day here but this is so c making of haners i got these home for five ninety nine a pack of five white gold hangers soers riight y dresses on definitely gets release i got this from amazon the cutt thing it's a little bing security box for our repair and it comes a little gar piece and sing glasses and that's like the cingcited about that this is another mazon mine i have tow flower girls and it came in a packg only ten dollars so those are so cute excited for the little flower girl baskets this is probablyveryinging this is our colors it's teractta and sea glass green and this is our wedding invitation which i've not shown anyone online so starts off super cute with this little gold seal which we got this up at s as well as the ribbonus simple blue notts from a craft store just to sticket down on the back is one of our engagement photos and so prettyor invitation is as kindly join us for the wedding ise arsm edwardst ells as a t become one and then i'm just using this little ribbon to cover the address and i'm so excited oh my goshs so pretty and yeah this is just a more details card that we told people the attire when to arrive all of that good stuff but isn't this just beatiful like this together with this looks so good ok running o time let me know to more details but so excited\"" ] }, "execution_count": 200, "metadata": {}, "output_type": "execute_result" } ], "source": [ "asr_prediction = transcribe(segments_manifest[92][\"uri\"])\n", "asr_prediction" ] }, { "cell_type": "code", "execution_count": 201, "id": "af5518c2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.6\n", "0.2857142857142857\n" ] } ], "source": [ "from suno_utils.utils.text_normalizer import (\n", " get_cer\n", ")\n", "\n", "print(get_cer(\"y'all\", \"you all\"))\n", "print(get_cer(\"y ' all\", \"you all\"))" ] }, { "cell_type": "code", "execution_count": 196, "id": "1af05bb8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"sixteen gigabytes of ram is a joke in two thousand and twenty two well i wouldn't exactly say that now i do personally have thirty two gigs of ram in my p c but for the average person who's just gaming or streaming you're not gonna need more than sixteen gigs sixteen gigabytes is more than enough for gaming and even i've never even gotten close to my thirty two gigabyte limit so in conclusion you don't really need thirty two gigs of ram sixteen will be just fine\"" ] }, "execution_count": 196, "metadata": {}, "output_type": "execute_result" } ], "source": [ "e = normalize(\n", " Tokens.from_dict(segments_manifest[110][\"transcript\"][\"tokens\"]),\n", " asr_prediction=asr_prediction,\n", " n_max_hyp=1000,\n", " n_max_tokens=50,\n", " custom_norm_rules=CUSTOM_NORM_RULES,\n", ")\n", "a = e.plaintext\n", "a" ] }, { "cell_type": "code", "execution_count": 88, "id": "3ebcf5ab", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'a two on two game'" ] }, "execution_count": 88, "metadata": {}, "output_type": "execute_result" } ], "source": [ "e = normalize(\n", " Tokens.from_text(\"a 2on2 game\"),\n", " asr_prediction=\"a two on two game\",\n", " n_max_hyp=1000,\n", " n_max_tokens=20,\n", " custom_norm_rules=CUSTOM_NORM_RULES,\n", ")\n", "e.plaintext" ] }, { "cell_type": "code", "execution_count": 42, "id": "92555768", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "an 150 airplane\n" ] }, { "data": { "text/plain": [ "{'an one five oh airplane',\n", " 'an one five zero airplane',\n", " 'an one hundred and fifty airplane',\n", " 'an one hundred fifty airplane'}" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "NORMALIZER.normalize(\n", " text=\"an 150 airplane\",\n", " verbose=False,\n", " n_tagged=200,\n", " punct_post_process=False,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "8c4e213a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ae757e34", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ce634d07", "metadata": {}, "outputs": [], "source": [ "f8446d0f-0b19-4da5-b9af-60d3da3abd2c\n", "\n", "# from suno_utils.web.rev import parse_transcript\n", "\n", "segment_transcripts = []\n", "for m in audio_rev_metas:\n", " srt_filepath = os.path.join(FROM_REV_DIR, m[\"uid\"] + \".srt\")\n", " if not os.path.exists(srt_filepath):\n", " print(\"not found, skipping...\")\n", " continue\n", " with open(srt_filepath) as f:\n", " transcript_srt = f.read()\n", " rev_transcript = parse_rev_captions(transcript_srt)\n", " n_segments = int(len(m[\"segments_meta\"]) / 2)\n", " token_segments = get_segments(rev_transcript, n_segments, global_indicator_speaker=False)\n", " segment_transcripts_chunk = []\n", " missing_ids = set(range(n_segments))\n", " for segment_idx, tokens in token_segments:\n", " segment_info = m[\"segments_meta\"][segment_idx * 2 + 1]\n", " segment_transcripts_chunk.append({\n", " \"uid\": segment_info[\"orginial_audio_uid\"],\n", " \"duration_s\": segment_info[\"duration_s\"],\n", " \"transcript\": tokens.anonymize_speakers(),\n", " })\n", " missing_ids -= set([segment_idx])\n", " missing_ids = sorted([str(n) for n in missing_ids])\n", " missing_str = \"\"\n", " if len(missing_ids) > 0:\n", " missing_str = \" (\" + \",\".join(missing_ids[:3])\n", " if len(missing_ids) > 3:\n", " missing_str += \"...\"\n", " missing_str += \")\"\n", " print(\"{}/{} segments retained in {}{}.\".format(\n", " len(segment_transcripts_chunk), n_segments, m[\"uid\"], missing_str\n", " ))\n", " segment_transcripts.extend(segment_transcripts_chunk)" ] }, { "cell_type": "code", "execution_count": null, "id": "d3d86de5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a6f077fb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d394bb5b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ff7c5d47", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }