{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "490dc40c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "3cce39dd", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['CUDA_VISIBLE_DEVICES'] = ''" ] }, { "cell_type": "code", "execution_count": 3, "id": "4f92bb44", "metadata": {}, "outputs": [], "source": [ "import re\n", "import pandas as pd\n", "import tqdm\n", "import json\n", "import sox\n", "import requests\n", "import wave\n", "\n", "DATA_DIR = \"/mnt/data-ssd-1/data/supreme_court/\"\n", "WAV_DIR = DATA_DIR + \"wavs/\"\n", "JSON_DIR = DATA_DIR + \"gentle_jsons/\"\n", "SLICE_DIR = DATA_DIR + \"slices/\"\n", "\n", "SAMPLE_RATE = 16_000\n", "BYTE_WIDTH = 2\n", "N_CHANNELS = 1\n", "\n", "def read_wav(fp):\n", " with wave.open(fp) as f:\n", " params = f.getparams()\n", " assert(params.nchannels == N_CHANNELS)\n", " assert(params.sampwidth == BYTE_WIDTH)\n", " assert(params.framerate == SAMPLE_RATE)\n", " audio_bytes = f.readframes(params.nframes)\n", " return audio_bytes\n", "\n", "def write_wav(fp, audio_bytes):\n", " with wave.open(fp, 'wb') as f:\n", " f.setnchannels(N_CHANNELS)\n", " f.setsampwidth(BYTE_WIDTH)\n", " f.setframerate(SAMPLE_RATE)\n", " f.writeframes(audio_bytes)\n", " return None" ] }, { "cell_type": "code", "execution_count": 4, "id": "08144376", "metadata": {}, "outputs": [], "source": [ "import portion\n", "\n", "# get speaker section annotations\n", "with open(DATA_DIR + \"sections.json\") as f:\n", " section_data = json.load(f)\n", "with open(DATA_DIR + \"diarization_meta.json\") as f:\n", " diarization_meta = json.load(f)\n", " \n", "SPEAKER_INTERVAL_LOOKUP = {}\n", "for uuid in set(section_data.keys()) & set(diarization_meta.keys()):\n", " assert(len(diarization_meta[uuid]) == len(section_data[uuid]))\n", " SPEAKER_INTERVAL_LOOKUP[uuid] = portion.IntervalDict()\n", " for (start_idx, end_idx), (speaker, _) in zip(diarization_meta[uuid], section_data[uuid]):\n", " SPEAKER_INTERVAL_LOOKUP[uuid][portion.closedopen(start_idx, end_idx)] = speaker" ] }, { "cell_type": "code", "execution_count": 5, "id": "191a4eae", "metadata": {}, "outputs": [], "source": [ "from pydub import AudioSegment\n", "from IPython.display import display\n", "import webrtcvad\n", "import portion\n", "\n", "VAD_FRAME_DURATION_MS = 10\n", "N_FRAME_BYTES = int(SAMPLE_RATE * VAD_FRAME_DURATION_MS / 1000) * BYTE_WIDTH\n", "\n", "def play_bytes(b, from_s=None, to_s=None):\n", " if to_s is not None:\n", " b = b[:round(int(to_s * BYTE_WIDTH * SAMPLE_RATE))]\n", " if from_s is not None:\n", " b = b[round(int(from_s * BYTE_WIDTH * SAMPLE_RATE)):]\n", " display(AudioSegment(\n", " b,\n", " frame_rate=SAMPLE_RATE,\n", " sample_width=BYTE_WIDTH,\n", " channels=N_CHANNELS,\n", " ))\n", " \n", "def _array_to_int(arr):\n", " if not isinstance(arr, np.ndarray):\n", " raise ValueError(\"only implemented for numpy arrays\")\n", " # enforce int16\n", " if arr.dtype == np.int16:\n", " pass\n", " elif arr.dtype in (np.float32, np.float64):\n", " # alert if signal too high\n", " if np.abs(arr).max() > 1:\n", " raise ValueError(\"signal overflow\")\n", " arr = (arr * 32768).clip(-32768, 32767).astype(np.int16)\n", " else:\n", " # TODO: convert other formats\n", " raise NotImplementedError(\"unknown format\")\n", " # alert if audio too loud\n", " if np.abs(arr).mean() / 32768 >= 0.1:\n", " raise ValueError(\"audio seems too loud\")\n", " return arr\n", "\n", "def play_array(arr, from_s=None, to_s=None):\n", " arr = _array_to_int(arr)\n", " play_bytes(arr.tobytes(), from_s=from_s, to_s=to_s)" ] }, { "cell_type": "code", "execution_count": 25, "id": "c3333d86", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "778 pairs ready!\n" ] } ], "source": [ "# get pairs\n", "min_align_frac = 0.5\n", "json_fps = {fp.split(\"/\")[-1][:-5]: JSON_DIR + fp for fp in os.listdir(JSON_DIR) if fp[-5:] == \".json\"}\n", "pairs = []\n", "failed_uuids = []\n", "for uuid, json_fp in tqdm.tqdm(json_fps.items()):\n", " # verify align fraction\n", " with open(json_fp) as f:\n", " d = json.load(f)\n", " align_frac = mean([e[\"case\"] == \"success\" for e in d[\"words\"] if e[\"case\"] != \"not-found-in-transcript\"])\n", " if align_frac < min_align_frac:\n", " failed_uuids.append(uuid)\n", " continue\n", " # compile pairs\n", " wav_fp = WAV_DIR + uuid + \".wav\"\n", " pairs.append((uuid, json_fps[uuid], wav_fp))\n", "print(len(pairs), \"pairs ready!\")" ] }, { "cell_type": "code", "execution_count": 27, "id": "d2b6c602", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('14-1375',\n", " '/mnt/data-ssd-1/data/supreme_court/gentle_jsons/14-1375.json',\n", " '/mnt/data-ssd-1/data/supreme_court/wavs/14-1375.wav')" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "uuid, json_fp, wav_fp = pairs[0]\n", "uuid, json_fp, wav_fp" ] }, { "cell_type": "code", "execution_count": null, "id": "880705db", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "71d1f9d4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "928aa20f", "metadata": {}, "source": [ "### get vad gaps" ] }, { "cell_type": "code", "execution_count": 34, "id": "9cd5428f", "metadata": {}, "outputs": [], "source": [ "DEFAULT_VAD_LEVEL = 2 # 3 is less speech, 0 is more\n", "\n", "def get_vad_gaps(wav_fp, min_gap_s=0.1, vad_level=DEFAULT_VAD_LEVEL):\n", " vad = webrtcvad.Vad(vad_level)\n", "\n", " audio_bytes = read_wav(wav_fp)\n", "\n", " frame_silences = []\n", " for n in range(len(audio_bytes) // N_FRAME_BYTES):\n", " frame_bytes = audio_bytes[n * N_FRAME_BYTES:(n + 1) * N_FRAME_BYTES]\n", " frame_silences.append(not vad.is_speech(frame_bytes, SAMPLE_RATE))\n", "\n", " n_silence_streak = int(ceil(min_gap_s * 1_000 / VAD_FRAME_DURATION_MS / 2))\n", "\n", " cur_n = 0\n", " lr_annot = []\n", " for frame_silence in frame_silences:\n", " is_valid = False\n", " if frame_silence:\n", " if cur_n >= n_silence_streak:\n", " is_valid = True\n", " cur_n += 1\n", " else:\n", " cur_n = 0\n", " lr_annot.append(is_valid)\n", "\n", " cur_n = 0\n", " rl_annot = []\n", " for frame_silence in frame_silences[::-1]:\n", " is_valid = False\n", " if frame_silence:\n", " if cur_n >= n_silence_streak:\n", " is_valid = True\n", " cur_n += 1\n", " else:\n", " cur_n = 0\n", " rl_annot.append(is_valid)\n", "\n", " vad_frame_gaps = [a and b for a, b in zip(lr_annot, rl_annot[::-1])]\n", "\n", " # accumulate to start stop index\n", " gap_spans_f = []\n", " cur_idx = None\n", " for n, is_valid in enumerate(vad_frame_gaps):\n", " if is_valid and cur_idx is None:\n", " cur_idx = n\n", " if not is_valid and cur_idx is not None:\n", " gap_spans_f.append((cur_idx, n))\n", " cur_idx = None\n", " if cur_idx is not None:\n", " gap_spans_f.append((cur_idx, len(vad_frame_gaps)))\n", " \n", " # convert to intervals and seconds\n", " gap_intervals = portion.closedopen(0, 0)\n", " for start_f, end_f in gap_spans_f:\n", " start_s = start_f * VAD_FRAME_DURATION_MS / 1_000\n", " end_s = end_f * VAD_FRAME_DURATION_MS / 1_000\n", " gap_intervals |= portion.closedopen(start_s, end_s)\n", " \n", " return gap_intervals\n", " \n", "vad_gap_intervals = get_vad_gaps(wav_fp)" ] }, { "cell_type": "code", "execution_count": null, "id": "64430573", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1ad94b2d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b0777373", "metadata": {}, "source": [ "### get align gaps" ] }, { "cell_type": "code", "execution_count": 35, "id": "a0b50aeb", "metadata": {}, "outputs": [], "source": [ "def load_align_meta(json_fp):\n", " with open(json_fp) as f:\n", " d = json.load(f)\n", " fulltext = d[\"transcript\"]\n", " align_info = []\n", " prev_end = 0\n", " for e in d[\"words\"]:\n", " if \"phones\" in e:\n", " del e[\"phones\"]\n", " if e[\"case\"] == \"not-found-in-transcript\":\n", " # skipping disfluencies for now because their start/end times overlap with words\n", " continue\n", " e[\"success\"] = e[\"case\"] == \"success\"\n", " del e[\"case\"]\n", " if \"alignedWord\" in e and e[\"alignedWord\"] == \"\":\n", " e[\"success\"] = False\n", " if \"start\" in e and \"end\" in e and e[\"end\"] - e[\"start\"] <= 0.02:\n", " e[\"success\"] = False\n", " if \"alignedWord\" in e:\n", " del e[\"alignedWord\"]\n", " if \"start\" in e:\n", " e[\"start\"] = round(e[\"start\"], 2)\n", " if \"end\" in e:\n", " e[\"end\"] = round(e[\"end\"], 2)\n", " if \"start\" in e and e[\"start\"] < prev_end:\n", " # sometimes a word will start before the previous one\n", " e[\"success\"] = False\n", " if \"end\" in e:\n", " prev_end = e[\"end\"]\n", " align_info.append(e)\n", " return fulltext, align_info\n", "\n", "def get_align_intervals(align_info, audio_dur_s, min_gap_s=0.1, safe_pad=2):\n", " gap_intervals = portion.closedopen(0, 0)\n", " for n in range(len(align_info) + 1):\n", " # check surroundings for success\n", " is_safe = True\n", " for nf in range(n, n + safe_pad):\n", " if nf >= len(align_info):\n", " continue\n", " is_safe = is_safe and align_info[nf][\"success\"]\n", " for nb in range(n - 1, n - safe_pad - 1, -1):\n", " if nb < 0:\n", " continue\n", " is_safe = is_safe and align_info[nb][\"success\"]\n", " if not is_safe:\n", " continue\n", " # check also for length of interval\n", " start_s = align_info[n - 1][\"end\"] if n >= 1 else 0\n", " end_s = align_info[n][\"start\"] if n < len(align_info) else audio_dur_s\n", " if end_s - start_s > min_gap_s:\n", " gap_intervals |= portion.closedopen(start_s + min_gap_s / 2, end_s - min_gap_s / 2)\n", " return gap_intervals" ] }, { "cell_type": "code", "execution_count": 36, "id": "3204ab5e", "metadata": {}, "outputs": [], "source": [ "fulltext, align_info = load_align_meta(json_fp)\n", "audio_dur_s = sox.file_info.duration(wav_fp)\n", "align_gap_intervals = get_align_intervals(align_info, audio_dur_s)" ] }, { "cell_type": "code", "execution_count": null, "id": "4b896e1d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f36216e7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e6f412cf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "1d3fbeb7", "metadata": {}, "source": [ "### make slices" ] }, { "cell_type": "code", "execution_count": 37, "id": "e9f03231", "metadata": {}, "outputs": [], "source": [ "INV_REPLACE_DICT = {\n", " chr(500): \"[laughter]\",\n", "}\n", "\n", "def _invert_text(text):\n", " for k, v in INV_REPLACE_DICT.items():\n", " text = text.replace(k, v)\n", " return text\n", "\n", "def _get_snippet_text(start_idx, end_idx, fulltext):\n", " text = fulltext[start_idx:end_idx]\n", " # prepend and append symbols\n", " for n in range(3):\n", " if end_idx + n >= len(fulltext) or fulltext[end_idx + n] not in \".?\\\"':;\":\n", " break\n", " text = text + fulltext[end_idx + n]\n", " for n in range(-1, -3, -1):\n", " if start_idx + n < 0 or fulltext[start_idx + n] not in \"\\\"'\":\n", " break\n", " text = fulltext[start_idx + n] + text\n", " text = _invert_text(text)\n", " return text\n", "\n", "def get_slices(fulltext, align_info, gap_intervals, speaker_lookup, audio_dur_s, max_dur_s=15.0, min_dur_s=2.0):\n", " # get valid slice intervals\n", " slice_intervals = []\n", " offs_s = gap_intervals.lower\n", " for _ in range(int(audio_dur_s // min_dur_s) + 1):\n", " if audio_dur_s - offs_s <= min_dur_s:\n", " break\n", " gaps_found = gap_intervals & portion.closedopen(offs_s + min_dur_s, offs_s + max_dur_s)\n", " if gaps_found.empty:\n", " remaining_interval = (gap_intervals & portion.closedopen(offs_s + min_dur_s, inf))\n", " if not remaining_interval.empty:\n", " offs_s = remaining_interval.lower\n", " continue\n", " break\n", " slice_intervals.append((offs_s, gaps_found.upper))\n", " offs_s = gaps_found.upper\n", " # annotate with text and speakers\n", " slices = [] # [(start, end), turns]\n", " for start_s, end_s in slice_intervals:\n", " start_idx = None\n", " for e in align_info:\n", " if \"start\" in e and e[\"start\"] >= start_s and start_idx == None:\n", " start_idx = e[\"startOffset\"]\n", " end_idx = None\n", " for e in align_info[::-1]:\n", " if \"end\" in e and e[\"end\"] <= end_s and end_idx == None:\n", " end_idx = e[\"endOffset\"]\n", " # one can be none if edge but not both\n", " assert(not start_idx == end_idx == None)\n", " \n", " speaker_intervals = []\n", " if start_idx != None and end_idx != None:\n", " for k, speaker in speaker_lookup[portion.closedopen(start_idx, end_idx)].items():\n", " for kk in k:\n", " speaker_intervals.append(((kk.lower, kk.upper), speaker))\n", " speaker_intervals = sorted(speaker_intervals, key=lambda k: k[0][0])\n", "\n", " turns = [] # [speaker, text]\n", " for (start_idx, end_idx), speaker in speaker_intervals:\n", " text = _get_snippet_text(start_idx, end_idx, fulltext)\n", " turns.append((speaker, text))\n", " \n", " slices.append(((round(start_s, 2), round(end_s, 2)), turns))\n", " return slices" ] }, { "cell_type": "code", "execution_count": 38, "id": "5c819125", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "66.5% of transcript resulted in slices.\n" ] } ], "source": [ "safe_gap_intervals = vad_gap_intervals & align_gap_intervals\n", "slices = get_slices(fulltext, align_info, safe_gap_intervals, SPEAKER_INTERVAL_LOOKUP[uuid], audio_dur_s)\n", "cov_frac = sum([e[1] - e[0] for e, _ in slices]) / audio_dur_s\n", "print(\"{}% of transcript resulted in slices.\".format(round(cov_frac * 100, 1)))\n", "if cov_frac < 0.5:\n", " raise ValueError(\"Only {}% of transcript resulted in slices.\".format(round(cov_frac * 100, 1)))" ] }, { "cell_type": "code", "execution_count": 39, "id": "218e2d4d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[((9.48, 19.98),\n", " [('CHIEF JUSTICE ROBERTS', 'Mr. Smith.'),\n", " ('MR. SMITH',\n", " 'Mr. Chief Justice, and may it please the Court: On the issue we initially asked this Court to resolve in the petition for certiorari, the parties are now in complete agreement.')]),\n", " ((19.98, 29.02),\n", " [('MR. SMITH',\n", " \"That issue, of course, was whether a prevailing defendant in a Title VII case is barred from seeking attorneys' fees if it hasn't prevailed on the merits.\")]),\n", " ((29.02, 43.84),\n", " [('MR. SMITH',\n", " \"As we showed in our opening brief, such a rule, which exists only in the Eighth Circuit, makes little sense. It doesn't -- it is certainly not compelled by the statutory language and doesn't serve any rationing statutory policy to take away the power of work fees in a -- in a\")])]" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "slices[:3]" ] }, { "cell_type": "code", "execution_count": 40, "id": "81101341", "metadata": {}, "outputs": [], "source": [ "audio_bytes = read_wav(wav_fp)" ] }, { "cell_type": "code", "execution_count": 41, "id": "f7bb3eeb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "MR. SMITH:\n", " add additional constraints. Thank you, Your Honor.\n", "\n", "CHIEF JUSTICE ROBERTS:\n", " Thank you, counsel. The case is submitted.\n", "\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "n = -1\n", "(start_s, end_s), turns = slices[n]\n", "for speaker, text in turns:\n", " print(speaker + \":\")\n", " print(\" \" + text)\n", " print()\n", "play_bytes(audio_bytes, start_s, end_s)" ] }, { "cell_type": "code", "execution_count": 17, "id": "08ddebb9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CHIEF JUSTICE ROBERTS:\n", " We'll now hear argument in Case 10-179, Stern v. Marshall. Mr. Richland.\n", "\n", "MR. RICHLAND:\n", " Mr. Chief Justice, and may it please the Court:\n", "\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "e = None\n", "for s in slices:\n", " if len(s[-1]) > 1:\n", " e = s\n", " break\n", "if e is not None:\n", " (start_s, end_s), turns = e \n", " for speaker, text in turns:\n", " print(speaker + \":\")\n", " print(\" \" + text)\n", " print()\n", " play_bytes(audio_bytes, start_s, end_s)" ] }, { "cell_type": "code", "execution_count": 18, "id": "63054afe", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "MR. ENGLERT:\n", " the Court could either -- [laughter]\n", "\n", "JUSTICE BREYER:\n", " I had to make that premise in order to --\n", "\n", "MR. ENGLERT:\n", " No, no, I understand. I understand, but given the premise, the Court could then either then reach an alternative ground for affirmance, which is well within the ordinary operation of this Court's rules or send it back.\n", "\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "e = None\n", "for s in slices:\n", " if \"[laughter]\" in \" \".join([ss[-1] for ss in s[-1]]):\n", " e = s\n", " break\n", "if e is not None:\n", " (start_s, end_s), turns = e \n", " for speaker, text in turns:\n", " print(speaker + \":\")\n", " print(\" \" + text)\n", " print()\n", " play_bytes(audio_bytes, start_s, end_s)" ] }, { "cell_type": "code", "execution_count": null, "id": "a192c1fa", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bacb256c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a8cafb56", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "5f1adf16", "metadata": {}, "source": [ "### optimize some params" ] }, { "cell_type": "code", "execution_count": 42, "id": "d2f67725", "metadata": {}, "outputs": [], "source": [ "import random\n", "\n", "random.seed(8006)\n", "\n", "test_pairs = pairs[:]\n", "random.shuffle(test_pairs)\n", "test_pairs = test_pairs[:10]" ] }, { "cell_type": "code", "execution_count": 44, "id": "5cd49b4a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "10it [00:23, 2.31s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "81.4% coverage\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "covs = []\n", "all_slices = []\n", "all_audio_bytes = []\n", "for n, (uuid, json_fp, wav_fp) in tqdm.tqdm(enumerate(test_pairs)):\n", " vad_gap_intervals = get_vad_gaps(wav_fp, vad_level=2, min_gap_s=0.1)\n", " fulltext, align_info = load_align_meta(json_fp)\n", " audio_dur_s = sox.file_info.duration(wav_fp)\n", " align_gap_intervals = get_align_intervals(align_info, audio_dur_s, min_gap_s=0.1)\n", " safe_gap_intervals = vad_gap_intervals & align_gap_intervals\n", " slices = get_slices(fulltext, align_info, safe_gap_intervals, SPEAKER_INTERVAL_LOOKUP[uuid], audio_dur_s)\n", " slices = [(n, s) for s in slices]\n", " all_slices.extend(slices)\n", " cov_frac = sum([e[1] - e[0] for _, (e, _) in slices]) / audio_dur_s\n", " covs.append(cov_frac)\n", " audio_bytes = read_wav(wav_fp)\n", " all_audio_bytes.append(audio_bytes)\n", "print(\"{}% coverage\".format(round(mean(covs) * 100, 1)))\n", "# 81.4% coverage - default" ] }, { "cell_type": "code", "execution_count": 87, "id": "b92d9104", "metadata": {}, "outputs": [], "source": [ "# # random\n", "# n_samples = 10\n", "# a = all_slices[:]\n", "# random.shuffle(a)\n", "# for n, ((start_s, end_s), turns) in a[:n_samples]:\n", "# for speaker, text in turns:\n", "# print(speaker + \":\")\n", "# print(\" \" + text)\n", "# play_bytes(all_audio_bytes[n], start_s, end_s)\n", "# print()" ] }, { "cell_type": "code", "execution_count": 86, "id": "4f83e1ef", "metadata": {}, "outputs": [], "source": [ "# # shortest audio\n", "# n_samples = 10\n", "# a = all_slices[:]\n", "# a = sorted(a, key=lambda k: k[1][0][1] - k[1][0][0])\n", "# for n, ((start_s, end_s), turns) in a[:n_samples]:\n", "# for speaker, text in turns:\n", "# print(speaker + \":\")\n", "# print(\" \" + text)\n", "# play_bytes(all_audio_bytes[n], start_s, end_s)\n", "# print()" ] }, { "cell_type": "code", "execution_count": 46, "id": "68121644", "metadata": {}, "outputs": [], "source": [ "# # shortest transcript\n", "# n_samples = 10\n", "# a = all_slices[:]\n", "# a = sorted(a, key=lambda k: len(\" \".join([e[-1] for e in k[-1][-1]])))\n", "# for n, ((start_s, end_s), turns) in a[:n_samples]:\n", "# for speaker, text in turns:\n", "# print(speaker + \":\")\n", "# print(\" \" + text)\n", "# play_bytes(all_audio_bytes[n], start_s, end_s)\n", "# print()" ] }, { "cell_type": "code", "execution_count": null, "id": "c537ff92", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4d0faf2e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "54221e8a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "035448d6", "metadata": {}, "source": [ "### make slices" ] }, { "cell_type": "code", "execution_count": 47, "id": "ce531213", "metadata": {}, "outputs": [], "source": [ "def _write_slices(pair, do_write=True):\n", " # obtain slice meta\n", " uuid, json_fp, wav_fp = pair\n", " vad_gap_intervals = get_vad_gaps(wav_fp)\n", " fulltext, align_info = load_align_meta(json_fp)\n", " audio_dur_s = sox.file_info.duration(wav_fp)\n", " align_gap_intervals = get_align_intervals(align_info, audio_dur_s)\n", " safe_gap_intervals = vad_gap_intervals & align_gap_intervals\n", " slices = get_slices(fulltext, align_info, safe_gap_intervals, SPEAKER_INTERVAL_LOOKUP[uuid], audio_dur_s)\n", " cov_frac = sum([e[1] - e[0] for e, _ in slices]) / audio_dur_s\n", " # write slices\n", " dir_fp = SLICE_DIR + uuid + \"/\"\n", " if do_write and os.path.exists(dir_fp):\n", " raise ValueError(\"directory already exists\")\n", " if do_write:\n", " os.mkdir(dir_fp)\n", " audio_bytes = read_wav(wav_fp)\n", " file_paths = []\n", " for (start_s, end_s), _ in slices:\n", " start_ms = int(start_s * 1_000)\n", " end_ms = int(end_s * 1_000)\n", " slice_fp = dir_fp + \"{}_{}.wav\".format(start_ms, end_ms)\n", " start_b = int(start_s * BYTE_WIDTH * SAMPLE_RATE)\n", " end_b = int(end_s * BYTE_WIDTH * SAMPLE_RATE)\n", " if do_write:\n", " write_wav(slice_fp, audio_bytes[start_b:end_b])\n", " file_paths.append(slice_fp)\n", " slice_metas = []\n", " for s, fp in zip(slices, file_paths):\n", " slice_metas.append((uuid, s, fp))\n", " return cov_frac, slice_metas\n", "\n", "def write_slices(pair, do_write=True):\n", " try:\n", " cov_frac, slice_metas = _write_slices(pair, do_write=do_write)\n", " except:\n", " cov_frac, slice_metas = 0, []\n", " return cov_frac, slice_metas" ] }, { "cell_type": "markdown", "id": "5b8ef64b", "metadata": {}, "source": [ "#### Test writing" ] }, { "cell_type": "code", "execution_count": 48, "id": "fecbbf7e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "66.5% of transcript resulted in slices.\n" ] } ], "source": [ "cov_frac, slice_metas = write_slices(pairs[0], do_write=False)\n", "print(\"{}% of transcript resulted in slices.\".format(round(cov_frac * 100, 1)))" ] }, { "cell_type": "code", "execution_count": 51, "id": "194bff8b", "metadata": {}, "outputs": [], "source": [ "# !ls /mnt/data-ssd-1/data/supreme_court/slices/" ] }, { "cell_type": "code", "execution_count": 50, "id": "84433f48", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('14-1375',\n", " ((9.48, 19.98),\n", " [('CHIEF JUSTICE ROBERTS', 'Mr. Smith.'),\n", " ('MR. SMITH',\n", " 'Mr. Chief Justice, and may it please the Court: On the issue we initially asked this Court to resolve in the petition for certiorari, the parties are now in complete agreement.')]),\n", " '/mnt/data-ssd-1/data/supreme_court/slices/14-1375/9480_19980.wav'),\n", " ('14-1375',\n", " ((19.98, 29.02),\n", " [('MR. SMITH',\n", " \"That issue, of course, was whether a prevailing defendant in a Title VII case is barred from seeking attorneys' fees if it hasn't prevailed on the merits.\")]),\n", " '/mnt/data-ssd-1/data/supreme_court/slices/14-1375/19980_29020.wav')]" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "slice_metas[:2]" ] }, { "cell_type": "markdown", "id": "39806eb9", "metadata": {}, "source": [ "### do writing" ] }, { "cell_type": "code", "execution_count": 55, "id": "7cb0762d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 778/778 [36:30<00:00, 2.82s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "621.0 hours.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# TODO: multicore\n", "all_slice_metas = []\n", "all_cov_fracs = []\n", "for pair in tqdm.tqdm(pairs):\n", " cov_frac, slice_metas = write_slices(pair, do_write=True)\n", " all_slice_metas.extend(slice_metas)\n", " all_cov_fracs.append(cov_frac)\n", "print(round(sum([(e[1][0][1] - e[1][0][0]) for e in all_slice_metas]) / 60 / 60, 1), \"hours.\")\n", "# 621.0 hours." ] }, { "cell_type": "code", "execution_count": 58, "id": "6945591f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('14-1375',\n", " ((9.48, 19.98),\n", " [('CHIEF JUSTICE ROBERTS', 'Mr. Smith.'),\n", " ('MR. SMITH',\n", " 'Mr. Chief Justice, and may it please the Court: On the issue we initially asked this Court to resolve in the petition for certiorari, the parties are now in complete agreement.')]),\n", " '/mnt/data-ssd-1/data/supreme_court/slices/14-1375/9480_19980.wav')" ] }, "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_slice_metas[0]" ] }, { "cell_type": "code", "execution_count": 52, "id": "bbdc908f", "metadata": {}, "outputs": [], "source": [ "# !ls /mnt/data-ssd-1/data/supreme_court/slices/\n", "# !rm -rf /mnt/data-ssd-1/data/supreme_court/slices/*" ] }, { "cell_type": "code", "execution_count": 57, "id": "b4b2c847", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"slice_meta_03_03_22.json\", \"w\") as f:\n", " json.dump(all_slice_metas, f)" ] }, { "cell_type": "markdown", "id": "734f931c", "metadata": {}, "source": [ "#### save global meta" ] }, { "cell_type": "code", "execution_count": 104, "id": "1e5f2dbc", "metadata": {}, "outputs": [], "source": [ "meta_df = pd.read_csv(DATA_DIR + \"scrape_meta.csv\")\n", "meta_df = meta_df.rename(columns={\"url_id\": \"uuid\"})\n", "meta_df = meta_df.drop(\"id\", axis=1)\n", "meta_df[\"date\"] = pd.to_datetime(meta_df[\"date\"]).dt.strftime(\"%Y-%m-%d\")\n", "file_df = pd.read_csv(DATA_DIR + \"raw_file_meta.csv\")\n", "meta_df[\"duration_s\"] = meta_df[\"uuid\"].map(file_df.set_index(\"uuid\")[\"duration_s\"])\n", "meta_df.to_json(DATA_DIR + \"meta_03_03_22.json\", orient=\"records\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a2620c28", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7dffc3f5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "95fe7628", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "62f816de", "metadata": {}, "outputs": [], "source": [ "# TODO: inject disfluencies\n", "# TODO: investigate long stretches of misalignment -> pdf parsing??" ] }, { "cell_type": "markdown", "id": "06d4b731", "metadata": {}, "source": [ "## Dataset ideas" ] }, { "cell_type": "markdown", "id": "ba33f6dc", "metadata": {}, "source": [ "- #### laughter in the courtroom\n", "- #### silence in the courtroom\n", "- #### numbers (III, 465, 879(a), ...)\n", "- #### talking over each other \n", "- #### hesitations\n", "- #### bad quality by using vad 3 vs 2\n", "- #### generals at court" ] }, { "cell_type": "markdown", "id": "40b44319", "metadata": {}, "source": [ "### Data TODOs\n", "- #### run gentle and vad alignment on snippets to filter words at ends, inject filler words and give word timestamps\n", "- #### add audio watermark" ] }, { "cell_type": "code", "execution_count": null, "id": "1ec92ff8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "171def2a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f632ad04", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 111, "id": "3e8796f2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['14-1375',\n", " [[9.48, 19.98],\n", " [['CHIEF JUSTICE ROBERTS', 'Mr. Smith.'],\n", " ['MR. SMITH',\n", " 'Mr. Chief Justice, and may it please the Court: On the issue we initially asked this Court to resolve in the petition for certiorari, the parties are now in complete agreement.']]],\n", " '/mnt/data-ssd-1/data/supreme_court/slices/14-1375/9480_19980.wav']" ] }, "execution_count": 111, "metadata": {}, "output_type": "execute_result" } ], "source": [ "with open(DATA_DIR + \"slice_meta_03_03_22.json\") as f:\n", " slice_data = json.load(f)\n", "slice_data[0]" ] }, { "cell_type": "code", "execution_count": 116, "id": "11a38438", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "192354" ] }, "execution_count": 116, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(slice_data)" ] }, { "cell_type": "code", "execution_count": 150, "id": "5d18fcf5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 found\n" ] } ], "source": [ "# check for covid\n", "query = r\"\\bSpider-Man\\b\"\n", "results = []\n", "for s in slice_data:\n", " if re.search(query, \" \".join([e[-1] for e in s[1][-1]])):\n", " results.append(s)\n", "print(len(results), \"found\")" ] }, { "cell_type": "code", "execution_count": 151, "id": "7a589a97", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[['18-877',\n", " [[1723.21, 1736.21],\n", " [['JUSTICE BREYER',\n", " 'What the state decides to do with its own website, charging $5 or something, is to run Rocky, Marvel, whatever, Spider-Man']]],\n", " '/mnt/data-ssd-1/data/supreme_court/slices/18-877/1723210_1736210.wav']]" ] }, "execution_count": 151, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results[:3]" ] }, { "cell_type": "code", "execution_count": null, "id": "dc068cbc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a590fd3a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c33a501b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 290, "id": "8a989497", "metadata": {}, "outputs": [], "source": [ "# check for laughter\n", "fpl = []\n", "for _, (_, turns), fp in all_slice_metas:\n", " if \"[laughter]\" in \" \".join([e[-1] for e in turns]):\n", " fpl.append((fp, turns))" ] }, { "cell_type": "code", "execution_count": 293, "id": "adb05cf0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[('CHIEF JUSTICE ROBERTS', 'simple rule and states could decide what they want to do. Correct? I see my time is up. I\\'d love to say \"correct\" to that. [laughter] Well, since -- I\\'d say correct and stop if I were you. I would least -- at least like to give you the final word. You can take a sentence.')]\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 293, "metadata": {}, "output_type": "execute_result" } ], "source": [ "n = 2\n", "print(fpl[n][1])\n", "AudioSegment.from_wav(fpl[n][0])" ] }, { "cell_type": "code", "execution_count": null, "id": "9d005bd4", "metadata": {}, "outputs": [], "source": [ "# check for laughter\n", "fpl = []\n", "for _, (_, turns), fp in all_slice_metas:\n", " if \"[laughter]\" in \" \".join([e[-1] for e in turns]):\n", " fpl.append((fp, turns))\n", "\n", "n = 2\n", "print(fpl[n][1])\n", "AudioSegment.from_wav(fpl[n][0])" ] }, { "cell_type": "code", "execution_count": null, "id": "ad1c2ace", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b9169d1f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1dbb0143", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8542a76a", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }