{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "c8c6684f", "metadata": {}, "outputs": [], "source": [ "from collections import Counter\n", "import json\n", "import os\n", "import random\n", "import re\n", "import time\n", "\n", "import bs4\n", "import editdistance\n", "import numpy as np\n", "from python_speech_features import mfcc\n", "import tqdm\n", "\n", "from suno_utils.utils.alignment import (\n", " align_segments, postprocess_slice_meta, prepare_alignment_maps, process_align_meta\n", ")\n", "from suno_utils.utils.conversion import Audio\n", "from suno_utils.utils.notebook import play_audio, load_ffmpeg, nemo_predict\n", "from suno_utils.utils.slicer import get_align_gaps, get_intervals_from_gaps, get_vad_gaps\n", "from suno_utils.utils.text import normalize_whitespace" ] }, { "cell_type": "code", "execution_count": 2, "id": "2cb7e1fc", "metadata": {}, "outputs": [], "source": [ "CORRUPT_UIDS = {\n", " \"20220216\", # corrupt audio \n", " \"20211118\", # missing files\n", " \"20210407\", # missing files\n", " \"20200702\", # missing files\n", " \"20200701\", # missing files\n", " \"20200603\", # missing files\n", " \"20200526\", # missing files\n", " \"20191205\", # missing files\n", " \"20191023\", # missing files\n", " \"20190508\", # missing files\n", " \"20190313\", # missing files\n", " \"20201125\", # weird meeting\n", "}\n", "\n", "DATA_DIR = \"/mnt/data-ssd-1/data/nz-parl/\"\n", "\n", "\n", "def _make_unique_list(input_list, key_f=None):\n", " el_seen = set()\n", " reduced_list = []\n", " for el in input_list:\n", " if key_f is None:\n", " k = el\n", " else:\n", " k = key_f(el)\n", " if k not in el_seen:\n", " reduced_list.append(el)\n", " el_seen.add(k)\n", " return reduced_list\n", "\n", "\n", "with open(os.path.join(DATA_DIR, \"manifest.json\")) as f:\n", " manifest = json.load(f)\n", " for m in manifest:\n", " m[\"id\"] = m[\"url\"].split(\"_\")[-1]\n", " assert(len(m[\"links\"][\"youtube\"]) == 0)\n", " manifest = _make_unique_list(manifest, key_f=lambda x: x[\"id\"])\n", " manifest = [m for m in manifest if m[\"id\"] not in CORRUPT_UIDS]\n", " print(len(manifest), \"events\")\n", " \n", "with open(\"vimeo_votes.json\") as f:\n", " VIDEO_VOTES = json.load(f)\n", " \n", "# 236 events" ] }, { "cell_type": "code", "execution_count": 3, "id": "1fc89cc5", "metadata": {}, "outputs": [], "source": [ "def _get_audio_overlap(audio_l, audio_r, stepsize_s=0.01, max_overlap_s=15, test_len_s=2):\n", " max_overlap_s = min(max_overlap_s, audio_l.duration_s)\n", " if audio_l.duration_s < test_len_s or audio_r.duration_s < test_len_s:\n", " return 0\n", " # prepare audio arrays for testing\n", " s_r = audio_r.get_slice(to_s=test_len_s)\n", " f_r = mfcc(s_r.array, samplerate=s_r.sample_rate, winlen=stepsize_s, winstep=stepsize_s, preemph=0)\n", " s_l = audio_l.get_slice(from_s=audio_l.duration_s-max_overlap_s)\n", " f_l = mfcc(s_l.array, samplerate=s_l.sample_rate, winlen=stepsize_s, winstep=stepsize_s, preemph=0)\n", " # check correlation of arrays with different shifts\n", " l = []\n", " for n in np.arange(test_len_s / stepsize_s, max_overlap_s / stepsize_s + 1, dtype=int):\n", " start_idx = int(f_l.shape[0] - n)\n", " end_idx = int(min(f_l.shape[0], f_l.shape[0] - n + f_r.shape[0]))\n", " f_l_s = f_l[start_idx:end_idx]\n", " c = np.corrcoef(f_l_s.ravel(), f_r[:len(f_l_s)].ravel())[0, 1]\n", " l.append((n, c))\n", " best_n, best_c = sorted(l, key=lambda x: x[-1])[-1]\n", " if best_c >= 0.8 and np.mean([e[-1] for e in l]) <= best_c - 0.1:\n", " return best_n * stepsize_s\n", " return 0\n", "\n", "\n", "def _concat_audios(audios):\n", " if len(audios) == 0:\n", " None\n", " total_audio_bytes = audios[0].bytes\n", " for audio_l, audio_r in zip(audios[:-1], audios[1:]):\n", " overlap_s = _get_audio_overlap(audio_l, audio_r)\n", " total_audio_bytes += audio_r.get_slice(from_s=overlap_s).bytes \n", " audio = Audio(total_audio_bytes, audios[0].sample_rate, audios[0].byte_width, audios[0].n_channels)\n", " return audio\n", "\n", "\n", "def assemble_full_audio(manifest_item):\n", " uid = manifest_item[\"id\"]\n", " # assemble video paths in order\n", " video_filepaths = []\n", " for link in manifest_item[\"links\"][\"vimeo\"]:\n", " v_uid = link.strip(\"/\").split(\"/\")[-1]\n", " # skip votes\n", " if VIDEO_VOTES[uid][v_uid]:\n", " continue\n", " filepath = DATA_DIR + uid + \"/\" + v_uid + \".mp4\"\n", " video_filepaths.append(filepath)\n", " # deduplicate, invert order, and remove intro section (usually prayer)\n", " video_filepaths = _make_unique_list(video_filepaths)[::-1][1:]\n", " # load video audio\n", " audios = [load_ffmpeg(filepath, sample_rate=44_100) for filepath in video_filepaths]\n", " # concat audios\n", " audio = _concat_audios(audios)\n", " return audio" ] }, { "cell_type": "code", "execution_count": 4, "id": "76e5e917", "metadata": {}, "outputs": [], "source": [ "def _clean_html_str(s):\n", " # TODO: do we want unidecode?\n", " s = re.sub(r\"\\<.+?\\>\", \" \", s)\n", " s = s.replace(\"—\", \" -- \").replace(\"…\", \" -- \")\n", " return normalize_whitespace(s)\n", "\n", "\n", "def _fixup_speaker_typos(sections):\n", " # count speaker occurences\n", " speaker_counts = Counter()\n", " for _, s, _ in sections:\n", " if s is None:\n", " continue\n", " speaker_counts[s] += 1\n", " # build mapper for mistakes\n", " speaker_name_fixup_map = {}\n", " potential_speaker_errors = [s for s, c in speaker_counts.items() if c == 1]\n", " for s in potential_speaker_errors:\n", " # if unambiguous fixup for editdistance 1 available then do it\n", " better_alternatives = []\n", " for sa, c in speaker_counts.items():\n", " if c > 1 and editdistance.eval(s, sa) == 1 and len(s) >= 5 and len(sa) >= 5:\n", " better_alternatives.append(sa)\n", " if len(better_alternatives) == 1:\n", " speaker_name_fixup_map[s] = better_alternatives[0]\n", " # do the fixup\n", " sections = [(st, speaker_name_fixup_map.get(s, s), t) for st, s, t in sections]\n", " return sections\n", "\n", "\n", "def _parse_speaker(s):\n", " # remove text that describes things we don't need\n", " s = re.sub(r\"\\s*to the\\s*\\.*$\", \"\", s).strip(\": \")\n", " s = re.sub(r\"\\s*on behalf of.*$\", \"\", s)\n", " # simplify\n", " s = _clean_html_str(s)\n", " # do some more trimming\n", " s = s.title()\n", " s = re.sub(r\"(\\bhon\\b)|(\\brt\\b)|(\\bdr\\b)\", \" \", s, flags=re.IGNORECASE)\n", " s = re.sub(r\"^[0-9\\.\\,\\s]+\", \"\", s)\n", " # treat parentheses correctly\n", " if s.count(\"(\") > s.count(\")\"):\n", " s = s + \")\" * (s.count(\"(\") - s.count(\")\")) # insert closing if missing\n", " elif s.count(\"(\") < s.count(\")\"):\n", " s = s.replace(\"(\", \" \").replace(\")\", \" \") # punt if opening missing\n", " s = re.sub(r\"(\\(.*)\\(.*\\)(.*\\))\", \"\\\\1\\\\2\", s) # remove doubles\n", " if re.search(r\"\\(.*\\)\", s) and (\n", " s.startswith(\"Chairperson\") or s.startswith(\"Assistant Speaker\") or \n", " s.startswith(\"Assistant Chairperson\") or s.startswith(\"Temporary Chairperson\")\n", " ):\n", " # handle things like \"Assistant Speaker ( Jenny Salesa) (Remote)\"\n", " p_items = re.findall(r\"\\(.*?\\)\", s)\n", " s = \" \".join([p_items[0][1:-1].strip()] + p_items[1:])\n", " # keep some\n", " s = re.sub(r\"\\(\\s*Remote\\s*\\)\", \" [remote] \", s)\n", " s = re.sub(r\"\\(\\s*(?:Labour|National)\\s*\\-\\-\\s*(.+)\\s*\\)\", \" [loc:\\\\1] \", s)\n", " # discard rest\n", " s = re.sub(r\"\\(.+?\\)\", \" \", s).strip(\": \")\n", " # split info\n", " speaker = normalize_whitespace(re.sub(r\"\\[.*?\\]\", \" \", s))\n", " speaker_meta = \" \".join(re.findall(r\"\\[.*?\\]\", s))\n", " return speaker, speaker_meta\n", "\n", "\n", "def parse_nz_parl_transcript(html_str):\n", " soup = bs4.BeautifulSoup(html_str)\n", " main_body = soup.find(\"div\", {\"class\": \"body-text--hansard\"})\n", " segments = []\n", " speaker_meta_map = {}\n", " cur_speaker = None\n", " cur_text = []\n", " for transcript_section in main_body.find_all(\"li\"):\n", " # skip vote sections\n", " if transcript_section.decode_contents().count(\"class=\\\"Vote\") >= 3:\n", " continue \n", " for p in transcript_section.find_all(\"p\"):\n", " s = p.decode_contents()\n", " # remove timestamp\n", " s = re.sub(r\"^\\\", \"\", s)\n", " # simplify strong\n", " s = re.sub(r\"\\<\\/strong\\>(\\s*)\\\", \"\\\\1\", s)\n", " # if all bold then header and skip\n", " if re.match(r\"^\\s*\\[^\\<\\>]*\\<\\/strong\\>\\s*$\", s):\n", " continue\n", " # find speaker\n", " speaker = cur_speaker\n", " m = re.search(r\"(^.*?\\<\\/strong\\>\\s*\\:)|(^.*?\\:\\s*\\<\\/strong\\>)\", s)\n", " if m:\n", " speaker = m.group()\n", " s = s[len(speaker):]\n", " speaker, speaker_meta = _parse_speaker(speaker)\n", " if len(speaker_meta) > 0:\n", " speaker_meta_map[speaker] = speaker_meta\n", " s = _clean_html_str(s)\n", " if (\n", " re.search(r\"^.{0,20} Speaker took the Chair at.{0,20}$\", s) or # header\n", " re.search(r\"^\\s*(?:Prayers|Karakia)\\s*.\\s*$\", s) or # header\n", " re.search(r\"(?:Amene?)\\s*.\\s*$\", s) or # prayer section\n", " re.search(r\"^\\s*\\[[^\\[\\]]*\\]\\s*$\", s) # translations\n", " ):\n", " continue\n", " if speaker != cur_speaker and len(cur_text) > 0:\n", " if cur_speaker is None:\n", " speaker_str = None\n", " else:\n", " speaker_str = (cur_speaker + \" \" + speaker_meta_map.get(cur_speaker, \"\")).strip()\n", " segments.append((\"speech\", speaker_str, normalize_whitespace(\" \".join(cur_text))))\n", " cur_text = []\n", " if len(s) > 0:\n", " cur_text.append(s)\n", " cur_speaker = speaker\n", " if len(cur_text) > 0:\n", " if cur_speaker is None:\n", " speaker_str = None\n", " else:\n", " speaker_str = (cur_speaker + \" \" + speaker_meta_map.get(cur_speaker, \"\")).strip()\n", " segments.append((\"speech\", speaker_str, normalize_whitespace(\" \".join(cur_text))))\n", " segments = _fixup_speaker_typos(segments)\n", " return segments" ] }, { "cell_type": "code", "execution_count": 5, "id": "f27be767", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "81.3% of text aligned\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "many vad gaps were found, try reducing aggressiveness\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "79.2% slice coverage with 440 slices\n" ] } ], "source": [ "manifest_item = manifest[130]\n", "\n", "# get audio\n", "audio = assemble_full_audio(manifest_item)\n", "audio_16khz = audio.convert(16_000, 2, 1)\n", "\n", "# parse segments\n", "html_str = manifest_item[\"transcript_page\"]\n", "segments = parse_nz_parl_transcript(html_str)\n", "\n", "# align audio & transcript\n", "fulltext, speaker_idx_map, meta_idx_map = prepare_alignment_maps(segments)\n", "align_dict = align_segments(audio_16khz, fulltext)\n", "align_meta = process_align_meta(align_dict[\"words\"])\n", "print(\"{}% of text aligned\".format(round(np.mean([e[\"success\"] for e in align_meta]) * 100, 1)))\n", "\n", "# make slices\n", "vad_gap_intervals = get_vad_gaps(audio_16khz, vad_level=3)\n", "align_gap_intervals = get_align_gaps(align_meta, audio.duration_s)\n", "slice_intervals = get_intervals_from_gaps(vad_gap_intervals, align_gap_intervals, max_unac_vad_tol_s=3.0)\n", "slices_meta = postprocess_slice_meta(\n", " slice_intervals, align_meta, fulltext, speaker_idx_map, meta_idx_map\n", ")\n", "slices_audio = [audio.get_slice(from_s=from_s, to_s=to_s) for (from_s, to_s), _ in slices_meta]\n", "slices_audio_16khz = [audio_16khz.get_slice(from_s=from_s, to_s=to_s) for (from_s, to_s), _ in slices_meta]\n", "cov_frac = np.sum([to_s - from_s for (from_s, to_s), _ in slices_meta]) / audio.duration_s\n", "print(\"{}% slice coverage with {} slices\".format(round(cov_frac * 100, 1), len(slices_meta)))\n", "# 77.9% of text aligned\n", "# 73.3% slice coverage with 546 slices" ] }, { "cell_type": "code", "execution_count": 6, "id": "4aae7ce3", "metadata": {}, "outputs": [], "source": [ "# TODO: run on larger sample, and improve actual slices (look at nemo preds too)\n", "# TODO: normalization improvements, add fst transformations like quote, speed?\n", "# TODO: nemo preds, slice filtering, normalization (plus improvements) and mfa" ] }, { "cell_type": "code", "execution_count": 9, "id": "c9d7bd17", "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "361\n", "[('David Seymour', \"unkind people might say that for political expediency, the Government abandoned that advice and introduced it, but I don't think it's a very good bill, and I'm not sure\")]\n", "\n", "unkind people might say for political expediency the government abandoned that advice and introduced it but i do't think it's a very good bill and i'm not sure\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "340\n", "[('Chlöe Swarbrick', 'The Greens are proud to support this legislation and hope that it is just the beginning of broader reforms to come.'), ('David Seymour', 'Well, thank you very much,')]\n", "\n", "mr spresident the greens are proud to support this legislation and hope that it is just the beginning of broader reforms to come david seml well thank you very much\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "106\n", "[('Winston Peters', 'I most certainly do, particularly the statement about how the previous National administration wilfully neglected the regions for nine long years.')]\n", "\n", "on behalf of the minister i most certainly do particularly the statement about how the previous national administration wilfully neglected the regions for nine long years the\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "152\n", "[('Gerry Brownlee [loc:Ilam]', 'to put conditions around a loan or the eventual transfer of equity is plain wrong.'), ('Speaker', 'OK. I think now for the third time,')]\n", "\n", "to put conditions around a loan or the eventual transfer of equity is plain wrong i think now for the third time\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "187\n", "[('Nicola Willis', 'at alert level 2?'), ('Chris Hipkins', \"What I can speak to is the very high-level principles that were set out in the Government's\")]\n", "\n", "at an alert level too to speak what i can speak to is the very high-level principles that were set out in the governments\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "273\n", "[('Stuart Nash', 'as they arise and to act swiftly. It could be applied for any time frame or procedural requirement across all tax or social policy obligations')]\n", "\n", "as they arise and to act swiftly it could be applied for any timeframe or procedural requirement across all tax or social policy obligations\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "217\n", "[('David Clark', \"I can't answer for those unnamed representatives the member speaks of, but what I would say is, of course, that the DHBs received over $200 million more\")]\n", "\n", "mr speaker i can't answer for se those unnamed representatives the member speaks of but what i would say is of course that the dhdbs received two hundred over two hundred million more\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "8\n", "[('Chris Hipkins', 'that we made in the last sitting block, extending the deadlines for replies to written questions and responses to')]\n", "\n", "that we made in the last sitting block extending the deadlines for replies to written questions and responses to\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "408\n", "[('Andrew Bayly [loc:Hunua]', \"If you look at the projections for the 2020 tax year, it was expected that corporate tax would generate $14.5 billion. So we've\")]\n", "\n", "if you look at the projections for the twenty twenty tax year it was expected that corporate tax would generate fourteen and a half billion so we\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "102\n", "[('Chris Hipkins', \"over quite a long period of time for things like a universal student allowance, in the current climate, it is really important that we channel additional financial support to the people who need it most in these economic times. So that's where the Government's priority is\")]\n", "\n", "over quite a long period of time for things like a universal student allowance in the current climate it is really important that we channel additional financial support to the people who need it most in these economic times and so that is where the government's priorities\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "for _ in range(10):\n", " n = random.choice(list(range(len(slices_meta))))\n", " m = slices_meta[n]\n", " a = slices_audio[n]\n", " print(n)\n", " print(m[1])\n", " print()\n", " print(nemo_predict(a))\n", " play_audio(a)" ] }, { "cell_type": "code", "execution_count": null, "id": "cc6d704e", "metadata": {}, "outputs": [], "source": [ "## manifest[0]\n", "# 00000 - buggy slice?" ] }, { "cell_type": "code", "execution_count": null, "id": "d070c1ad", "metadata": {}, "outputs": [], "source": [ "# TODO: try cycling through 8khz\n", "# TODO: figure out why [150] takes so long even after align" ] }, { "cell_type": "code", "execution_count": null, "id": "007f3186", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7e1bfad3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "77f4aa07", "metadata": {}, "outputs": [], "source": [ "# nemo predicts 'mr president' instead of 'mr speaker'" ] }, { "cell_type": "code", "execution_count": null, "id": "31beaf0e", "metadata": {}, "outputs": [], "source": [ "# Notes:\n", "# 'Mr. Chairman' often ignored\n", "# 'Mr. Speaker' often ignored\n", "# \"I call on the minister...\" ignored\n", "# 'Madam Chair' -> \"Madam Speaker\"\n", "# \"Hon\" -> \"Honorable\"\n", "# \"House resumed.\" slightly differently said\n", "# \"in proposed new section 141EE\" differently said\n", "# \"Progress to be reported.\" in text not pronounced\n", "# repetitions and disfluencies ignored\n", "# ( -> 'brackets' in norm\n", "# \" -> 'quote' in norm\n", "# offence - spelling\n", "# \"on the Finance and Expenditure Committee\" -> \"the FEC\"" ] }, { "cell_type": "code", "execution_count": null, "id": "049fe446", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "36425cad", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "64594c68", "metadata": {}, "source": [ "## do for all" ] }, { "cell_type": "code", "execution_count": 33, "id": "5fed9991", "metadata": {}, "outputs": [], "source": [ "to_data_dir = \"/mnt/data-ssd-1/data/nz-parl_georg/\"" ] }, { "cell_type": "code", "execution_count": null, "id": "3a7a3f3e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "16it [5:00:12, 1277.97s/it]many vad gaps were found, try reducing aggressiveness\n", "17it [5:18:37, 1225.98s/it]many vad gaps were found, try reducing aggressiveness\n", "21it [6:46:26, 1273.22s/it]many vad gaps were found, try reducing aggressiveness\n", "22it [7:15:24, 1412.47s/it]many vad gaps were found, try reducing aggressiveness\n", "24it [7:50:57, 1185.95s/it]many vad gaps were found, try reducing aggressiveness\n", "26it [9:03:13, 1634.23s/it]many vad gaps were found, try reducing aggressiveness\n", "28it [9:48:25, 1514.92s/it]many vad gaps were found, try reducing aggressiveness\n", "30it [10:22:43, 1216.32s/it]many vad gaps were found, try reducing aggressiveness\n", "31it [11:12:17, 1743.59s/it]many vad gaps were found, try reducing aggressiveness\n", "34it [12:25:38, 1564.11s/it]many vad gaps were found, try reducing aggressiveness\n", "37it [13:27:19, 1356.84s/it]many vad gaps were found, try reducing aggressiveness\n", "44it [14:38:04, 490.36s/it] many vad gaps were found, try reducing aggressiveness\n", "45it [14:46:09, 488.95s/it]many vad gaps were found, try reducing aggressiveness\n", "48it [15:47:00, 865.67s/it] many vad gaps were found, try reducing aggressiveness\n", "49it [16:14:34, 1101.91s/it]many vad gaps were found, try reducing aggressiveness\n", "51it [16:53:59, 1102.16s/it]many vad gaps were found, try reducing aggressiveness\n", "54it [17:53:13, 1065.32s/it]many vad gaps were found, try reducing aggressiveness\n", "56it [18:46:12, 1322.51s/it]" ] } ], "source": [ "failed_ids = []\n", "debug_info = {}\n", "slices_info = {}\n", "for n_id, manifest_item in tqdm.tqdm(enumerate(manifest)):\n", " try:\n", " t0 = time.time()\n", " # prepare paths\n", " to_segments_filepath = to_data_dir + \"segments/{}.json\".format(manifest_item[\"id\"])\n", " to_align_filepath = to_data_dir + \"align/{}.json\".format(manifest_item[\"id\"])\n", " to_slice_dir = to_data_dir + \"slices/{}/\".format(manifest_item[\"id\"])\n", " if not os.path.exists(os.path.dirname(to_segments_filepath)):\n", " os.makedirs(os.path.dirname(to_segments_filepath))\n", " if not os.path.exists(os.path.dirname(to_align_filepath)):\n", " os.makedirs(os.path.dirname(to_align_filepath))\n", " if not os.path.exists(os.path.dirname(to_slice_dir)):\n", " os.makedirs(os.path.dirname(to_slice_dir))\n", " \n", " # get audio\n", " audio = assemble_full_audio(manifest_item)\n", " audio_16khz = audio.convert(16_000, 2, 1)\n", "\n", " if os.path.exists(to_segments_filepath) and os.path.exists(to_align_filepath):\n", " # load segments and alignment\n", " with open(to_segments_filepath) as f:\n", " segments = json.load(f)\n", " with open(to_align_filepath) as f:\n", " align_dict = json.load(f)\n", " fulltext, speaker_idx_map, meta_idx_map = prepare_alignment_maps(segments)\n", " align_meta = process_align_meta(align_dict[\"words\"])\n", " else:\n", " # parse segments\n", " html_str = manifest_item[\"transcript_page\"]\n", " segments = parse_nz_parl_transcript(html_str)\n", " # align audio & transcript\n", " fulltext, speaker_idx_map, meta_idx_map = prepare_alignment_maps(segments)\n", " align_dict = align_segments(audio_16khz, fulltext)\n", " align_meta = process_align_meta(align_dict[\"words\"])\n", " with open(to_segments_filepath, \"w\") as f:\n", " json.dump(segments, f)\n", " with open(to_align_filepath, \"w\") as f:\n", " json.dump(align_dict, f)\n", "\n", " # make slices\n", " vad_gap_intervals = get_vad_gaps(audio_16khz, vad_level=3)\n", " align_gap_intervals = get_align_gaps(align_meta, audio.duration_s)\n", " slice_intervals = get_intervals_from_gaps(vad_gap_intervals, align_gap_intervals, max_unac_vad_tol_s=3.0)\n", " slices_meta = postprocess_slice_meta(\n", " slice_intervals, align_meta, fulltext, speaker_idx_map, meta_idx_map\n", " )\n", " \n", " # write slices and meta\n", " for n, ((from_s, to_s), _) in enumerate(slices_meta):\n", " to_wav_filepath = to_slice_dir + \"{}.wav\".format(n)\n", " if not os.path.exists(to_wav_filepath):\n", " audio_16khz.get_slice(from_s=from_s, to_s=to_s).write_wav(to_wav_filepath)\n", " \n", " # save info\n", " time_elapsed_s = round(time.time() - t0, 1)\n", " slices_duration_s = np.sum([to_s - from_s for (from_s, to_s), _ in slices_meta])\n", " cov_frac = slices_duration_s / audio.duration_s\n", " debug_info[manifest_item[\"id\"]] = {\n", " \"time_elapsed_s\": time_elapsed_s,\n", " \"text_align_pct\": round(np.mean([e[\"success\"] for e in align_meta]) * 100, 1),\n", " \"slice_cov_pct\": round(cov_frac * 100, 1),\n", " \"n_slices\": len(slices_meta),\n", " \"slices_duration_s\": round(slices_duration_s, 1),\n", " }\n", " slices_info[manifest_item[\"id\"]] = slices_meta\n", " \n", " # save progress\n", " with open(to_data_dir + \"debug_info.json\", \"w\") as f:\n", " json.dump(debug_info, f)\n", " with open(to_data_dir + \"slices_info.json\", \"w\") as f:\n", " json.dump(slices_info, f)\n", " \n", " except:\n", " failed_ids.append(n_id)" ] }, { "cell_type": "code", "execution_count": null, "id": "9b06683f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "daa61b14", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "68cc7aae", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "a0b6c0b8", "metadata": {}, "source": [ "## Get video speaker meta" ] }, { "cell_type": "code", "execution_count": 22, "id": "1d96da97", "metadata": {}, "outputs": [], "source": [ "import requests\n", "import bs4\n", "import tqdm\n", "import time\n", "\n", "\n", "HEADERS = {\n", " \"User-Agent\": (\n", " \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) \"\n", " \"Chrome/50.0.2661.102 Safari/537.36\"\n", " )\n", "}\n", "\n", "BASE_URL = \"https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_\"\n", "\n", "\n", "def _get_pages_to_follow(soup):\n", " pages_to_follow = []\n", " for link in soup.findAll(\"a\"):\n", " if not link.has_attr('href'):\n", " continue\n", " href = link[\"href\"]\n", " if href.startswith(\"?page=\"):\n", " pages_to_follow.append(href)\n", " return _make_unique_list(pages_to_follow)\n", "\n", "\n", "def _get_speaker_map(soup):\n", " speaker_map = {}\n", " for media_item in soup.findAll(\"li\", {\"class\": \"media\"}):\n", " media_html = media_item.decode_contents()\n", " media_text = media_item.text\n", " if \"vimeo\" in media_html:\n", " uid = re.search(r\"vimeo.com/([0-9]+)\\b\", media_html).group(1)\n", " speaker = re.search(r\"People\\:\\s*([\\w\\W]*?)\\s*State\\:\", media_item.text).group(1)\n", " speaker = speaker.strip(\", \")\n", " speaker = re.sub(r\"\\s{2,}\", \"; \", speaker)\n", " speaker_map[uid] = speaker\n", " return speaker_map\n", "\n", "def get_full_speaker_map(uid):\n", " video_url = BASE_URL + \"{}_{}/tab/video\".format(uid, uid)\n", " res = requests.get(video_url, headers=HEADERS)\n", " soup = bs4.BeautifulSoup(res.text)\n", " pages_to_follow = _get_pages_to_follow(soup) \n", "\n", " speaker_map = _get_speaker_map(soup)\n", " for page in pages_to_follow:\n", " time.sleep(random.random() * 0.5)\n", " res = requests.get(video_url + page, headers=HEADERS)\n", " soup = bs4.BeautifulSoup(res.text)\n", " for k, v in _get_speaker_map(soup).items():\n", " speaker_map[k] = v\n", " \n", " return speaker_map" ] }, { "cell_type": "code", "execution_count": null, "id": "c96b5d8b", "metadata": {}, "outputs": [], "source": [ "DATA_DIR = \"/mnt/data-ssd-1/data/nz-parl/\"\n", "\n", "with open(os.path.join(DATA_DIR, \"manifest.json\")) as f:\n", " manifest_tmp = json.load(f)\n", "\n", "speaker_data = {}\n", "failed_uids = []\n", "for m in tqdm.tqdm(manifest_tmp):\n", " uid = m[\"url\"].split(\"_\")[-1]\n", " if uid in speaker_data:\n", " continue\n", " try:\n", " speaker_data[uid] = get_full_speaker_map(uid)\n", " except:\n", " failed_uids.append(uid)\n", "print(len(failed_uids), \"failed\")" ] }, { "cell_type": "code", "execution_count": 64, "id": "a62742d4", "metadata": {}, "outputs": [], "source": [ "with open(\"vimeo_speakers.json\", \"w\") as f:\n", " json.dump(speaker_data, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "687e2158", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a9a1cad5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f9d2d93c", "metadata": {}, "source": [ "## identify vote in videos" ] }, { "cell_type": "code", "execution_count": 33, "id": "b02a220f", "metadata": {}, "outputs": [], "source": [ "from PIL import Image\n", "import io\n", "import re\n", "import ffmpeg\n", "import pytesseract\n", "import tqdm\n", "\n", "from suno_utils.utils.notebook import probe_ffmpeg\n", "\n", "\n", "def _check_video_for_vote(filepath):\n", " # check which frame we want to use\n", " meta = probe_ffmpeg(filepath, codec_type=\"video\")\n", " tot_frames = float(meta[\"duration\"]) * np.max([int(n) for n in meta[\"avg_frame_rate\"].split(\"/\")])\n", " # generate thumbnail image at ~1 minute into the video\n", " stream = ffmpeg.input(filepath)\n", " stream = stream.filter(\"select\", \"gte(n,{})\".format(min(2000, int(tot_frames*0.8))))\n", " stream = stream.filter(\"crop\", \"in_w*0.3\", \"in_h*0.1\", \"(in_w-in_w*0.7)/2\", \"in_h-in_h*0.2\")\n", " stream = stream.filter(\"scale\", \"in_w*1.3\", \"in_h\")\n", " stream = stream.output(\"pipe:\", vframes=1, format=\"image2\", vcodec=\"mjpeg\")\n", " out, _ = stream.run(capture_stdout=True, overwrite_output=True, quiet=True)\n", " # check OCR in image\n", " img = Image.open(io.BytesIO(out))\n", " if \"partyvote\" in re.sub(r\"\\s\", \"\", pytesseract.image_to_string(img)).lower():\n", " return True\n", " return False" ] }, { "cell_type": "code", "execution_count": 34, "id": "c2f6ad1f", "metadata": {}, "outputs": [], "source": [ "with open(\"vimeo_speakers.json\") as f:\n", " speaker_data = json.load(f)" ] }, { "cell_type": "code", "execution_count": 35, "id": "afb22ed3", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 246/246 [07:22<00:00, 1.80s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "0 failed\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "votes_data = {}\n", "failed_ids = []\n", "for manifest_item in tqdm.tqdm(manifest):\n", " uid = manifest_item[\"id\"]\n", " try:\n", " d = {}\n", " for link in manifest_item[\"links\"][\"vimeo\"]:\n", " v_uid = link.strip(\"/\").split(\"/\")[-1]\n", " if v_uid in d:\n", " continue\n", " # only check ones that don't have a speaker\n", " if len(speaker_data[uid][v_uid]) == 0:\n", " filepath = DATA_DIR + \"{}/{}.mp4\".format(uid, v_uid)\n", " if os.path.exists(filepath):\n", " d[v_uid] = _check_video_for_vote(filepath)\n", " else:\n", " d[v_uid] = False\n", " votes_data[uid] = d \n", " except:\n", " failed_ids.append(uid)\n", " pass\n", "print(len(failed_ids), \"failed\")" ] }, { "cell_type": "code", "execution_count": 37, "id": "413b6984", "metadata": {}, "outputs": [], "source": [ "with open(\"vimeo_votes.json\", \"w\") as f:\n", " json.dump(votes_data, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "3b7eaf39", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "900e3630", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "0b9eb4e4", "metadata": {}, "source": [ "## speaker locations" ] }, { "cell_type": "code", "execution_count": 189, "id": "c9a0ecc2", "metadata": {}, "outputs": [], "source": [ "speakers = []\n", "for manifest_item in manifest:\n", " html_str = manifest_item[\"transcript_page\"]\n", " segments = parse_nz_parl_transcript(html_str)\n", " segments = _fixup_speaker_typos(segments)\n", " speakers.extend([s for _, s, _ in segments if s is not None])\n", "\n", "unique_speakers = list(set([re.sub(r\"\\s*\\[.*?\\]\\s*\", \"\", s).strip() for s in speakers]))\n", "unique_speakers = [s for s in unique_speakers if \"speaker\" not in s.lower() and len(s.split()) >= 2]" ] }, { "cell_type": "code", "execution_count": 282, "id": "207ecbf0", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 178/178 [04:57<00:00, 1.67s/it]\n" ] } ], "source": [ "import requests\n", "import urllib.parse\n", "import time\n", "import tqdm\n", "\n", "HEADERS = {\n", " \"User-Agent\": (\n", " \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) \"\n", " \"Chrome/50.0.2661.102 Safari/537.36\"\n", " )\n", "}\n", "\n", "sess = requests.Session()\n", "\n", "speaker_birth_map = {}\n", "for speaker in tqdm.tqdm(unique_speakers):\n", " if \"speaker\" in s.lower() or len(s.split()) < 2:\n", " continue\n", " # find page that corresponds to name\n", " search_url = \"https://en.wikipedia.org/w/api.php\"\n", " query_ptn = \"{} new zealand\"\n", " params = {\n", " \"action\": \"query\",\n", " \"format\": \"json\",\n", " \"list\": \"search\",\n", " \"srsearch\": query_ptn.format(speaker)\n", " }\n", " res = sess.get(url=search_url, params=params, headers=HEADERS)\n", " results = res.json()['query']['search']\n", " if len(results) == 0:\n", " print(speaker)\n", " continue\n", " best_page_title = results[0][\"title\"]\n", " # grab birthplace\n", " page_url_ptn = (\n", " \"http://en.wikipedia.org/w/api.php?action=query&prop=revisions&rvprop=content&format=json\"\n", " \"&titles={}&rvsection=0&rvparse\"\n", " )\n", " res = sess.get(url=page_url_ptn.format(urllib.parse.quote(best_page_title)), headers=HEADERS)\n", " out = res.json()\n", " page_one = next(iter(out['query']['pages'].values()))\n", " revisions = page_one.get('revisions', [])\n", " infobox_html = next(iter(revisions[0].values()))\n", " m = re.search(\"Born.{0,300}?\\]*?href\\=[\\'\\\"]\\/wiki\\/([^\\'\\\"]*?)[\\'\\\"][^\\>]*?\\>[^\\<]*?\\<\\/a\\>\", infobox_html)\n", " if m and \" 0:\n", " word_aligment.append(((t.xmin, t.xmax), t.text))\n", " return word_aligment\n", "\n", "\n", "def mfa_align(audios, transcripts, num_cores=1):\n", " if len(audios) != len(transcripts):\n", " raise ValueError(\"need audios and transcripts that match\")\n", " with tempfile.TemporaryDirectory() as temp_dir:\n", " pairs_dir = os.path.join(temp_dir, \"pairs\")\n", " out_dir = os.path.join(temp_dir, \"out\")\n", " os.mkdir(pairs_dir)\n", " # make one folder for each core\n", " for n in range(num_cores):\n", " sub_pairs_dir = os.path.join(pairs_dir, \"{}\".format(n))\n", " os.mkdir(sub_pairs_dir)\n", " for n, (audio, transcript) in enumerate(zip(audios, transcripts)):\n", " assigned_core = n % num_cores\n", " sub_pairs_dir = os.path.join(pairs_dir, \"{}\".format(assigned_core))\n", " with open(os.path.join(sub_pairs_dir, \"{}.txt\".format(n)), \"w\") as f:\n", " f.write(transcript)\n", " audio.write_wav(os.path.join(sub_pairs_dir, \"{}.wav\".format(n)))\n", " cmd = MFA_PYTHON_CMD_PTN.format(\n", " conda_sh_filepath=CONDA_SH_FILEPATH, num_cores=num_cores, pairs_dir=pairs_dir, out_dir=out_dir\n", " )\n", " out_code = subprocess.call(cmd, shell=True)\n", " if out_code != 0:\n", " raise ValueError(\"mfa error\")\n", " word_aligments = []\n", " for n in range(len(audios)):\n", " assigned_core = n % num_cores\n", " sub_out_dir = os.path.join(out_dir, \"{}\".format(assigned_core))\n", " textgrid_filepath = os.path.join(sub_out_dir, \"{}.TextGrid\".format(n))\n", " if not os.path.exists(textgrid_filepath):\n", " word_aligments.append(None)\n", " continue\n", " word_aligment = _load_textgrid_file(textgrid_filepath)\n", " word_aligments.append(word_aligment)\n", " return word_aligments" ] }, { "cell_type": "code", "execution_count": 158, "id": "6716d14a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 338 ms, sys: 214 ms, total: 552 ms\n", "Wall time: 1min\n" ] } ], "source": [ "%%time\n", "audios = slices_audio_16khz\n", "transcripts = [\" \".join([text for _, text in turns]) for _, turns in slices_meta]\n", "word_aligments = mfa_align(audios, transcripts, num_cores=1)" ] }, { "cell_type": "code", "execution_count": 157, "id": "3bd128d1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 227 ms, sys: 192 ms, total: 419 ms\n", "Wall time: 20 s\n" ] } ], "source": [ "%%time\n", "audios = slices_audio_16khz\n", "transcripts = [\" \".join([text for _, text in turns]) for _, turns in slices_meta]\n", "word_aligments = mfa_align(audios, transcripts, num_cores=16)" ] }, { "cell_type": "code", "execution_count": null, "id": "4c945f0d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1ab5314e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "477fd39b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "374b5d2d", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 205, "id": "3b68ec59", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 205, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from pydub import AudioSegment\n", "AudioSegment.from_mp3(\"/mnt/data-ssd-1/data/nz-parl/20220317/audio/689097996.mp3\")" ] }, { "cell_type": "code", "execution_count": 206, "id": "207b9903", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 206, "metadata": {}, "output_type": "execute_result" } ], "source": [ "AudioSegment.from_mp3(\"/mnt/data-ssd-1/data/nz-parl/20220317/audio/689099481.mp3\")" ] }, { "cell_type": "code", "execution_count": 27, "id": "8d3c8601", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2019\n", "83 https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210317_20210317\n" ] } ], "source": [ "for n, manifest_item in enumerate(manifest):\n", " html_str = manifest_item[\"transcript_page\"]\n", " soup = bs4.BeautifulSoup(html_str)\n", " main_body = soup.find(\"div\", {\"class\": \"body-text--hansard\"})\n", " first_section = main_body.find_all(\"li\")[0]\n", " if len(first_section.decode_contents()) > 2000:\n", " print(len(first_section.decode_contents()))\n", " print(n, manifest_item[\"url\"])\n", " break\n", "# first_section\n", "# # skip prayers\n", "# if \"BeginningOfDay\" in first_section.decode_contents():\n", "# continue" ] }, { "cell_type": "code", "execution_count": null, "id": "8e2061a3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "52c2408a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4b65caad", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "063a677f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "200b63c7", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }