{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "64d773c6", "metadata": {}, "outputs": [], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": null, "id": "fe6f477a", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['CUDA_VISIBLE_DEVICES'] = '1'" ] }, { "cell_type": "code", "execution_count": null, "id": "0d497284", "metadata": {}, "outputs": [], "source": [ "import json\n", "import tempfile\n", "\n", "from multiprocessing import Pool\n", "import tqdm\n", "\n", "from suno_utils.utils.text import normalize_whitespace" ] }, { "cell_type": "code", "execution_count": null, "id": "d080fb5b", "metadata": {}, "outputs": [], "source": [ "DATA_DIR = \"/mnt/data-ssd-1/data/supreme_corpus/processed_data/\"\n", "\n", "with open(DATA_DIR + \"slices_meta.jsonl\") as f:\n", " slice_data = [json.loads(e) for e in f]" ] }, { "cell_type": "markdown", "id": "c5a9a3b3", "metadata": {}, "source": [ "### create 16khz versions" ] }, { "cell_type": "code", "execution_count": null, "id": "37bf4aaa", "metadata": {}, "outputs": [], "source": [ "# make dirs\n", "for uuid in set([e[\"uuid\"] for e in slice_data]):\n", " fd = DATA_DIR + \"slices_16khz/\" + uuid\n", " if not os.path.exists(fd):\n", " os.makedirs(fd)" ] }, { "cell_type": "code", "execution_count": null, "id": "158ce385", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.conversion import convert_audio\n", "\n", "def do_conversion(from_fp):\n", " to_fp = from_fp.replace(\"slices/\", \"slices_16khz/\")\n", " convert_audio(from_fp, to_fp, sample_rate=16_000)" ] }, { "cell_type": "code", "execution_count": null, "id": "79f2bd90", "metadata": {}, "outputs": [], "source": [ "fp_list = [DATA_DIR + e[\"path\"] for e in slice_data]" ] }, { "cell_type": "code", "execution_count": null, "id": "a4dc0e96", "metadata": {}, "outputs": [], "source": [ "# ~10 mins for 600h at 32 cores\n", "p = Pool(32)\n", "_ = p.map(do_conversion, fp_list)\n", "p.close()\n", "p.join()" ] }, { "cell_type": "markdown", "id": "f2d6ac58", "metadata": {}, "source": [ "### do slice alignments" ] }, { "cell_type": "code", "execution_count": null, "id": "3f12e927", "metadata": {}, "outputs": [], "source": [ "for e in tqdm.tqdm(slice_data):\n", " text = \" \".join([ee[\"text\"] for ee in e[\"transcript\"]])\n", " text = text.replace(\"[laughter]\", \" \").replace(\"--\", \" \")\n", " text = normalize_whitespace(text)\n", " fp = DATA_DIR + e[\"path\"].replace(\"slices/\", \"slices_16khz/\")[:-4] + \".txt\"\n", " with open(fp, \"w\") as f:\n", " f.write(text)" ] }, { "cell_type": "code", "execution_count": null, "id": "a0f7dcdd", "metadata": {}, "outputs": [], "source": [ "mfa align \\\n", " --clean \\\n", " -t /tmp \\\n", " /home/georg/notebooks/datasets/supreme_court/tmp/align_pairs \\\n", " english \\\n", " english \\\n", " /home/georg/notebooks/datasets/supreme_court/tmp/align_out \\\n", " --retry_beam 4000" ] }, { "cell_type": "code", "execution_count": null, "id": "597aa18a", "metadata": {}, "outputs": [], "source": [ "# conda activate mfa\n", "\n", "# rm -rf /mnt/data-ssd-1/data/supreme_corpus/processed_data/slice_alignments\n", "\n", "# mfa align \\\n", "# --clean \\\n", "# -t /tmp \\\n", "# -j 32 \\\n", "# /mnt/data-ssd-1/data/supreme_corpus/processed_data/slices_16khz \\\n", "# english \\\n", "# english \\\n", "# /mnt/data-ssd-1/data/supreme_corpus/processed_data/slice_alignments \\\n", "# --retry_beam 4000" ] }, { "cell_type": "code", "execution_count": null, "id": "c46a5745", "metadata": {}, "outputs": [], "source": [ "import textgrids\n", "\n", "slice_aligments = []\n", "failed_fps = []\n", "for e in tqdm.tqdm(slice_data):\n", " fp = DATA_DIR + \"slice_alignments/\" + e[\"path\"].replace(\"slices/\", \"\")[:-4] + \".TextGrid\"\n", " # check if alignment failed cause empty\n", " with open(DATA_DIR + e[\"path\"].replace(\"slices/\", \"slices_16khz/\")[:-4] + \".txt\") as f:\n", " text = f.read().strip()\n", " if len(text) == 0:\n", " slice_aligments.append([])\n", " continue\n", " # check if alignment failed\n", " elif not os.path.exists(fp):\n", " failed_fps.append(e[\"path\"])\n", " slice_aligments.append([])\n", " continue\n", " grid = textgrids.TextGrid(fp)\n", " word_aligments = []\n", " for t in grid[\"words\"]:\n", " if len(t.text) > 0:\n", " word_aligments.append(((t.xmin, t.xmax), t.text))\n", " slice_aligments.append(word_aligments)\n", "assert(len(failed_fps) == 0)" ] }, { "cell_type": "code", "execution_count": null, "id": "12d95db5", "metadata": {}, "outputs": [], "source": [ "# with open(DATA_DIR + \"slice_aligments.json\", \"w\") as f:\n", "# json.dump(slice_aligments, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "0466c06c", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"slice_aligments.json\") as f:\n", " slice_aligments = json.load(f)" ] }, { "cell_type": "markdown", "id": "6d483584", "metadata": {}, "source": [ "### make nemo predictions" ] }, { "cell_type": "code", "execution_count": null, "id": "cde7d57e", "metadata": {}, "outputs": [], "source": [ "import nemo.collections.asr as nemo_asr\n", "asr_model = nemo_asr.models.EncDecRNNTBPEModel.from_pretrained(model_name=\"stt_en_contextnet_1024\")\n", "# asr_model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(model_name=\"stt_en_conformer_ctc_large\")" ] }, { "cell_type": "code", "execution_count": null, "id": "f8d653f2", "metadata": {}, "outputs": [], "source": [ "out = asr_model.transcribe([\n", " DATA_DIR + e[\"path\"].replace(\"slices/\", \"slices_16khz/\")\n", " for e in slice_data\n", "], batch_size=32)\n", "pred_texts = out[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "bbfeca74", "metadata": {}, "outputs": [], "source": [ "# with open(DATA_DIR + \"nemo_preds.json\", \"w\") as f:\n", "# json.dump(pred_texts, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "2a091747", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"nemo_preds.json\") as f:\n", " pred_texts = json.load(f)" ] }, { "cell_type": "markdown", "id": "b5882a27", "metadata": {}, "source": [ "### normalize" ] }, { "cell_type": "code", "execution_count": null, "id": "8ec92487", "metadata": {}, "outputs": [], "source": [ "from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio\n", "import string\n", "from nemo.collections.nlp.data.text_normalization.utils import post_process_punct\n", "\n", "allowed_chars = set(string.ascii_lowercase + \" '\")\n", "\n", "def _normalize_asr_safe(text):\n", " text = \"\".join([c if c in allowed_chars else \" \" for c in text.lower()])\n", " text = normalize_whitespace(text)\n", " return text\n", "\n", "def _pre_normalize(text):\n", " text = text.replace(\"[laughter]\", \" \").replace(\"--\", \" \")\n", " text = normalize_whitespace(text)\n", " return text\n", "\n", "def _normalize_pred(pred_text):\n", " pred_text = \" \".join([w for w in pred_text.split() if w not in (\"uh\", \"um\")])\n", " pred_text = pred_text.replace(\"-\", \" \").replace(\"'\", \" ' \")\n", " pred_text = normalize_whitespace(pred_text)\n", " return pred_text\n", "\n", "l = [\n", " (\"Case No.\", \"Case\"),\n", "]\n", "# TODO: contractions don't become expanded\n", "with open(\"/tmp/custom_norm.tsv\", \"w\") as f:\n", " f.write(\"\\n\".join([\"\\t\".join(e) for e in l]) + \"\\n\")\n", "\n", "nemo_normalizer = NormalizerWithAudio(\n", " lang=\"en\",\n", " input_case=\"cased\",\n", " overwrite_cache=True,\n", " cache_dir=\"/tmp/nemo_fst\",\n", " whitelist=\"/tmp/custom_norm.tsv\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "8c2fc099", "metadata": {}, "outputs": [], "source": [ "def do_nemo_norm(combo):\n", " text_list, pred_raw = combo\n", " pred = _normalize_pred(pred_raw)\n", " if len(text_list) == 1 and len(text_list[0]) == 0:\n", " return [\"\"]\n", " options = [(\"\", [])]\n", " for text in text_list:\n", " text = _pre_normalize(text)\n", " normalized_texts = nemo_normalizer.normalize(\n", " text=text, \n", " verbose=False, \n", " n_tagged=-1, # TODO: maybe can do high instead\n", " punct_post_process=False,\n", " )\n", " new_options = []\n", " seen_txt = set()\n", " for ot, ol in options:\n", " for nt in normalized_texts:\n", " # TODO: mistake?: AT&T -> AT & T -> at t\n", " # maybe fix with known normalization dictionaries in symbols\n", " nnt = _normalize_asr_safe(nt)\n", " sep = \" \" if len(ot) > 0 else \"\"\n", " s = ot + sep + nnt\n", " if s not in seen_txt:\n", " seen_txt.add(s)\n", " else:\n", " continue\n", " idx_l = ol + [len(nnt.split())]\n", " new_options.append((s, idx_l))\n", " options = new_options\n", " # TODO: stop if this exploded?\n", " normalized_text, _ = nemo_normalizer.select_best_match(\n", " normalized_texts=[o for o, _ in options],\n", " input_text=options[0][0], # default return if pred is \"\", TODO: take most common?\n", " pred_text=pred,\n", " verbose=False,\n", " remove_punct=True,\n", " )\n", " correct_idx_map = [idx_map for o, idx_map in options if o == normalized_text]\n", " assert(len(correct_idx_map) == 1)\n", " correct_idx_map = correct_idx_map[0]\n", " norm_list = []\n", " offs = 0\n", " assert(len(correct_idx_map) == len(text_list))\n", " for n, raw_text in zip(correct_idx_map, text_list):\n", " s = \" \".join(normalized_text.split()[offs:offs + n])\n", " s = post_process_punct(raw_text, s)\n", " norm_list.append(s)\n", " offs += n\n", " assert(offs == len(normalized_text.split()))\n", " return len(options), norm_list" ] }, { "cell_type": "code", "execution_count": null, "id": "5c082106", "metadata": {}, "outputs": [], "source": [ "combo_list = [\n", " ([ee[\"text\"] for ee in e[\"transcript\"]], pred) \n", " for e, pred in zip(slice_data, pred_texts)\n", "]" ] }, { "cell_type": "code", "execution_count": null, "id": "44566ef7", "metadata": {}, "outputs": [], "source": [ "# ~1s per file per core, ~3.2h for full corpus at 60 cores\n", "p = Pool(60)\n", "t0 = time.time()\n", "normed_text_info = p.map(do_nemo_norm, combo_list, chunksize=10)\n", "n_norm_options, normed_texts = zip(*normed_text_info)\n", "t1 = time.time()\n", "print(round((t1 - t0) / 60 / 60, 1), \"hours\")\n", "p.close()\n", "p.join()" ] }, { "cell_type": "code", "execution_count": null, "id": "bbdd5353", "metadata": {}, "outputs": [], "source": [ "# with open(DATA_DIR + \"normed_text_info.json\", \"w\") as f:\n", "# json.dump(normed_text_info, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "1ed1f4ef", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"normed_text_info.json\") as f:\n", " normed_text_info = json.load(f)" ] }, { "cell_type": "markdown", "id": "a3aa2f10", "metadata": {}, "source": [ "### add disfluencies" ] }, { "cell_type": "code", "execution_count": null, "id": "fd11abfe", "metadata": {}, "outputs": [], "source": [ "# TODO: roll into nemo norm" ] }, { "cell_type": "code", "execution_count": null, "id": "339a083e", "metadata": {}, "outputs": [], "source": [ "from difflib import SequenceMatcher\n", "\n", "def fixup_insert_tokens(text_raw, normed_sections, text_pred):\n", " \n", " if len(normed_sections) == 0:\n", " return []\n", " \n", " am = [s.split() for s in normed_sections]\n", " al = [len(l) for l in am]\n", "\n", " # make flat token to original position map\n", " # make token affine to end of line instead of beginning (disfluencies)\n", " # flat_token_idx: (section_n, rel_token_idx)\n", " token_idx_map = {0: (0, 0)}\n", " n = 1\n", " for nn, l in enumerate(am):\n", " for nnn in range(1, len(l) + 1):\n", " token_idx_map[n] = (nn, nnn)\n", " n += 1\n", "\n", " a = \" \".join(normed_sections).split()\n", " b = text_pred.split()\n", " s = SequenceMatcher(None, a, b)\n", " for tag, i1, i2, j1, j2 in s.get_opcodes()[::-1]:\n", " if tag == \"insert\" and i1 == i2:\n", " insert_tokens = []\n", " if \"quote\" in b[j1:j2] and \"\\\"\" in text_raw:\n", " tokens = re.findall(r\"\\bquote on quote\\b|\\bquote\\b|\\bum\\b|\\buh\\b\", \" \".join(b[j1:j2]))\n", " insert_tokens = []\n", " for t in tokens:\n", " insert_tokens.extend(t.split())\n", " elif \"um\" in b[j1:j2] or \"uh\" in b[j1:j2]:\n", " insert_tokens = re.findall(r\"\\bum\\b|\\buh\\b\", \" \".join(b[j1:j2]))\n", " if len(insert_tokens) > 0:\n", " section_idx, rel_token_idx = token_idx_map[i1]\n", " am[section_idx] = am[section_idx][:rel_token_idx] + insert_tokens + am[section_idx][rel_token_idx:]\n", "\n", " fixed_sections = []\n", " for l in am:\n", " fixed_sections.append(\" \".join(l))\n", " \n", " return fixed_sections" ] }, { "cell_type": "code", "execution_count": null, "id": "31984ac3", "metadata": {}, "outputs": [], "source": [ "normed_fixed_texts = []\n", "for n, (e, (_, normed_sections), text_pred) in tqdm.tqdm(enumerate(zip(slice_data, normed_text_info, pred_texts))):\n", " text_raw = \" \".join([ee[\"text\"] for ee in e[\"transcript\"]])\n", " fixed_norm_sections = fixup_insert_tokens(text_raw, normed_sections, text_pred)\n", " normed_fixed_texts.append(fixed_norm_sections)" ] }, { "cell_type": "code", "execution_count": null, "id": "055f5f27", "metadata": {}, "outputs": [], "source": [ "# with open(DATA_DIR + \"normed_fixed_texts.json\", \"w\") as f:\n", "# json.dump(normed_fixed_texts, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "104e6ea4", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"normed_fixed_texts.json\") as f:\n", " normed_fixed_texts = json.load(f)" ] }, { "cell_type": "markdown", "id": "35688d0f", "metadata": {}, "source": [ "### redo slice alignment with normed texts" ] }, { "cell_type": "code", "execution_count": null, "id": "7e14cc8e", "metadata": {}, "outputs": [], "source": [ "for sections, e in tqdm.tqdm(zip(normed_fixed_texts, slice_data), total=len(slice_data)):\n", " fp = DATA_DIR + e[\"path\"].replace(\"slices/\", \"slices_16khz/\")[:-4] + \".txt\"\n", " with open(fp, \"w\") as f:\n", " f.write(\" \".join(sections))" ] }, { "cell_type": "code", "execution_count": null, "id": "c808db01", "metadata": {}, "outputs": [], "source": [ "# conda activate mfa\n", "\n", "# rm -rf /mnt/data-ssd-1/data/supreme_corpus/processed_data/slice_alignments_normed\n", "\n", "# mfa align \\\n", "# --clean \\\n", "# -t /tmp \\\n", "# -j 32 \\\n", "# /mnt/data-ssd-1/data/supreme_corpus/processed_data/slices_16khz \\\n", "# english \\\n", "# english \\\n", "# /mnt/data-ssd-1/data/supreme_corpus/processed_data/slice_alignments_normed \\\n", "# --retry_beam 4000" ] }, { "cell_type": "code", "execution_count": null, "id": "c212752b", "metadata": {}, "outputs": [], "source": [ "import textgrids\n", "\n", "token_metas_normed = []\n", "failed_fps = []\n", "for e in tqdm.tqdm(slice_data):\n", " fp = DATA_DIR + \"slice_alignments_normed/\" + e[\"path\"].replace(\"slices/\", \"\")[:-4] + \".TextGrid\"\n", " # check if alignment failed cause empty\n", " with open(DATA_DIR + e[\"path\"].replace(\"slices/\", \"slices_16khz/\")[:-4] + \".txt\") as f:\n", " text = f.read().strip()\n", " if len(text) == 0:\n", " token_metas_normed.append([])\n", " continue\n", " # check if alignment failed\n", " elif not os.path.exists(fp):\n", " failed_fps.append(e[\"path\"])\n", " token_metas_normed.append([])\n", " continue\n", " grid = textgrids.TextGrid(fp)\n", " tokens = []\n", " for t in grid[\"words\"]:\n", " if len(t.text) > 0:\n", " tokens.append(((t.xmin, t.xmax), t.text))\n", " token_metas_normed.append(tokens)\n", "assert(len(failed_fps) == 0)" ] }, { "cell_type": "code", "execution_count": null, "id": "a47b212f", "metadata": {}, "outputs": [], "source": [ "# with open(DATA_DIR + \"slice_alignments_normed.json\", \"w\") as f:\n", "# json.dump(token_metas_normed, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "7c9a0bd2", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"slice_alignments_normed.json\") as f:\n", " token_metas_normed = json.load(f)" ] }, { "cell_type": "markdown", "id": "ba4daa21", "metadata": {}, "source": [ "### filter slices for final corpus" ] }, { "cell_type": "code", "execution_count": null, "id": "14c7be73", "metadata": {}, "outputs": [], "source": [ "from difflib import SequenceMatcher\n", "\n", "def _check_safe_boundary(true, pred):\n", " is_safe = True\n", " a = true.split()\n", " b = pred.split()\n", " s = SequenceMatcher(None, a, b)\n", " op_codes = s.get_opcodes()\n", " if len(op_codes) < 2:\n", " return is_safe\n", " # check start\n", " tag, i1, i2, j1, j2 = op_codes[0]\n", " tag_2, i1_2, i2_2, j1_2, j2_2 = op_codes[1]\n", " if tag in (\"insert\", \"delete\") and tag_2 == \"equal\":# and j2_2 - j1_2 >= 2:\n", " is_safe = False\n", " # check end\n", " tag, i1, i2, j1, j2 = op_codes[-1]\n", " tag_2, i1_2, i2_2, j1_2, j2_2 = op_codes[-2]\n", " if tag in (\"insert\", \"delete\") and tag_2 == \"equal\":# and j2_2 - j1_2 >= 2:\n", " is_safe = False\n", " return is_safe\n", "\n", "from nemo.collections.asr.metrics.wer import word_error_rate\n", "\n", "def _clean_str_for_eval(s):\n", " s = re.sub(r\"\\s*\\bum\\b\\s*|\\s*\\buh\\b\\s*\", \" \", s)\n", " s = normalize_whitespace(s)\n", " return s\n", "\n", "def _get_cer(a, b):\n", " a = _clean_str_for_eval(a)\n", " b = _clean_str_for_eval(b)\n", " if a == b:\n", " return 0\n", " cer = word_error_rate([a], [b], use_cer=True)\n", " return cer" ] }, { "cell_type": "code", "execution_count": null, "id": "0f496ba3", "metadata": {}, "outputs": [], "source": [ "blocklist = set()\n", "for n, (e, pred, normed_sections, (n_norm_options, _)) in tqdm.tqdm(\n", " enumerate(zip(slice_data, pred_texts, normed_fixed_texts, normed_text_info)), total=len(slice_data)\n", "):\n", " text_raw = \" \".join([ee[\"text\"] for ee in e[\"transcript\"]])\n", " text = \" \".join(normed_sections)\n", " if len(pred) == 0 and len(text) > 0 and len(text) < 20:\n", " blocklist.add(n)\n", " continue\n", " if len(text) == 0 and len(pred) > 0:\n", " blocklist.add(n)\n", " continue\n", " # filter where pred boundary insertions or deletions\n", " if not _check_safe_boundary(text, pred):\n", " blocklist.add(n)\n", " continue\n", " # some phrases aren't usually transcribed\n", " if \"court is now adjourned\" in pred and \"court is now adjourned\" not in text:\n", " blocklist.add(n)\n", " continue\n", " # if cer is high and norm ambiguous we can't trust norm\n", " cer = _get_cer(text, pred)\n", " if cer >= 0.3 and n_norm_options > 1:\n", " blocklist.add(n)\n", " continue\n", "# known failures\n", "blocklist |= set([97611, 80141, 152227, 180812])\n", "print(\"{}% of slices filtered.\".format(round(len(blocklist) / len(slice_data) * 100, 1)))" ] }, { "cell_type": "code", "execution_count": null, "id": "2090a7cd", "metadata": {}, "outputs": [], "source": [ "# with open(DATA_DIR + \"blocklist.json\", \"w\") as f:\n", "# json.dump(list(blocklist), f)" ] }, { "cell_type": "code", "execution_count": null, "id": "377bc2da", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"blocklist.json\") as f:\n", " blocklist = set(json.load(f))" ] }, { "cell_type": "markdown", "id": "229feb57", "metadata": {}, "source": [ "## Prep for dataset viewer" ] }, { "cell_type": "code", "execution_count": null, "id": "c3e0cabf", "metadata": {}, "outputs": [], "source": [ "import json\n", "\n", "DATA_DIR = \"/mnt/data-ssd-1/data/supreme_corpus/processed_data/\"\n", " \n", "with open(DATA_DIR + \"slices_meta.jsonl\") as f:\n", " slice_data = [json.loads(e) for e in f]\n", " \n", "with open(DATA_DIR + \"normed_fixed_texts.json\") as f:\n", " true_texts = [\" \".join(e) for e in json.load(f)]\n", " \n", "with open(DATA_DIR + \"nemo_preds.json\") as f:\n", " pred_texts = json.load(f)" ] }, { "cell_type": "code", "execution_count": null, "id": "78fac8e2", "metadata": {}, "outputs": [], "source": [ "dl = []\n", "with open(\"tmp_dataset.jsonl\", \"w\") as f:\n", " for n, (pred, true, e) in enumerate(zip(pred_texts, true_texts, slice_data)):\n", " if n in blocklist:\n", " continue\n", " if len(true) == 0 or len(pred) == 0:\n", " continue\n", " d = {\n", " \"audio_filepath\": DATA_DIR + e[\"path\"].replace(\"slices/\", \"slices_16khz/\"), # (path to audio file)\n", " \"duration\": e[\"timestamp\"][1] - e[\"timestamp\"][0], # (duration of the audio file in seconds)\n", " \"text\": true, # (reference transcript)\n", " \"pred_text\": pred, # (ASR transcript)\n", " }\n", " dl.append(d)\n", " f.write(json.dumps(d) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "id": "8b04d8de", "metadata": {}, "outputs": [], "source": [ "# python data_explorer.py \\\n", "# /home/georg/notebooks/datasets/supreme_court/tmp_dataset.jsonl" ] }, { "cell_type": "markdown", "id": "bdc1ba94", "metadata": {}, "source": [ "## make final corpus" ] }, { "cell_type": "code", "execution_count": null, "id": "ff8b5c35", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "DATA_DIR = \"/mnt/data-ssd-1/data/supreme_corpus/processed_data/\"\n", "\n", "with open(DATA_DIR + \"slices_meta.jsonl\") as f:\n", " base_meta_list = [json.loads(e) for e in f]\n", "\n", "with open(DATA_DIR + \"blocklist.json\") as f:\n", " blocklist = set(json.load(f))\n", " \n", "with open(DATA_DIR + \"normed_fixed_texts.json\") as f:\n", " texts_normed_list = json.load(f)\n", "\n", "with open(DATA_DIR + \"slice_aligments.json\") as f:\n", " token_timings_list = json.load(f)\n", " \n", "with open(DATA_DIR + \"slice_alignments_normed.json\") as f:\n", " normed_token_timings_list = json.load(f)\n", " \n", "meta_df = pd.read_json(\"/mnt/data-ssd-1/data/supreme_corpus/raw_data/scrape_meta.jsonl\", lines=True)\n", "case_date_map = meta_df.set_index(\"uuid\")[\"date\"].dt.strftime(\"%Y/%m/%d\").to_dict()\n", "case_title_map = meta_df.set_index(\"uuid\")[\"title\"].to_dict()\n", "\n", "speaker_df = pd.read_csv(DATA_DIR.replace(\"processed_data/\", \"raw_data/\") + \"manual_speaker_meta.csv\")\n", "speaker_df = speaker_df.rename(columns={\n", " \"Supreme Corpus Alias\": \"speaker\",\n", " \"Full Name\": \"full_name\",\n", " \"birthday\": \"born_date\",\n", " \"Race\": \"reported_race\",\n", " \"Born (State)\": \"born_state\",\n", " \"Gender\": \"reported_gender\",\n", " \"Religion\": \"reported_religion\",\n", " \"Appointed By (President)\": \"appointed_by_president\",\n", " \"Appointed By (Party)\": \"appointed_by_party\",\n", " \"wiki page\": \"wiki_page\",\n", "})\n", "assert(speaker_df[\"speaker\"].nunique() == speaker_df.shape[0])\n", "speaker_df[\"reported_gender\"] = speaker_df[\"reported_gender\"].str.lower()\n", "assert(set(speaker_df[\"reported_gender\"].unique()) == set([np.nan, \"male\", \"female\"]))\n", "speaker_meta_map = speaker_df.set_index(\"speaker\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a915ef37", "metadata": {}, "outputs": [], "source": [ "def _norm_speaker_name(s):\n", " return re.sub(r\"[A-Z][A-Z]+\", lambda m: m.group(0).capitalize(), s)\n", "\n", "def _find_first_idx(partial_str, my_list, lowercase=True):\n", " idx_found = -1\n", " for n, e in enumerate(my_list):\n", " ef = e.lower() if lowercase else e\n", " if partial_str in ef:\n", " idx_found = n\n", " break\n", " return idx_found\n", "\n", "def _get_imputed_ts(idx, idx_ts_map, duration_s):\n", " lower_b = 0\n", " for n in range(idx - 1, -1, -1):\n", " if n in idx_ts_map:\n", " lower_b = idx_ts_map[n][-1]\n", " break\n", " upper_b = duration_s\n", " for n in range(idx + 1, np.max(list(idx_ts_map.keys())) + 1):\n", " if n in idx_ts_map:\n", " upper_b = idx_ts_map[n][0]\n", " break\n", " return (lower_b, upper_b)\n", "\n", "def get_timestamped_segments(sections, word_alignments, duration_s):\n", " tokens = []\n", " tokens_per_segment = []\n", " for s in sections:\n", " t = s.split()\n", " tokens.extend(t)\n", " tokens_per_segment.append(len(t))\n", " if len(tokens) == 0:\n", " return []\n", " # find aligned token in text and make timestamp map\n", " offs = 0\n", " idx_ts_map = {}\n", " for (ts, te), s in word_alignments:\n", " idx = _find_first_idx(s, tokens[offs:])\n", " idx_ts_map[offs + idx] = (ts, te)\n", " offs += idx + 1\n", " assert(-1 not in idx_ts_map)\n", " # make sure all tokens have a timestamp\n", " fixed_idx_ts_map = idx_ts_map.copy()\n", " for n in range(len(tokens)):\n", " if n not in idx_ts_map:\n", " imputed_ts = _get_imputed_ts(n, idx_ts_map, duration_s)\n", " fixed_idx_ts_map[n] = imputed_ts\n", " \n", " # collect together timestamped tokens\n", " timestamped_segments = []\n", " offs = 0\n", " for n_token in tokens_per_segment:\n", " timestamped_words = []\n", " for n in range(offs, offs + n_token):\n", " assert(n in fixed_idx_ts_map)\n", " timestamped_words.append((tokens[n], fixed_idx_ts_map[n]))\n", " offs += n_token\n", " timestamped_segments.append(timestamped_words)\n", " return timestamped_segments" ] }, { "cell_type": "code", "execution_count": null, "id": "f47a88d2", "metadata": {}, "outputs": [], "source": [ "slice_meta_list = []\n", "n_slice = 0\n", "for (\n", " n, (base_meta, texts_normed, token_timings, normed_token_timings)\n", ") in tqdm.tqdm(enumerate(zip(\n", " base_meta_list, texts_normed_list, token_timings_list, normed_token_timings_list\n", ")), total=len(slice_data)):\n", " if n in blocklist:\n", " continue\n", " duration_s = base_meta[\"timestamp\"][1] - base_meta[\"timestamp\"][0]\n", " # get token level timestamps\n", " texts = [e[\"text\"] for e in base_meta[\"transcript\"]]\n", " token_metas = get_timestamped_segments(texts, token_timings, duration_s)\n", " normed_token_metas = get_timestamped_segments(texts_normed, normed_token_timings, duration_s)\n", " # prepare segments\n", " segments = []\n", " for segment_info, token_meta in zip(base_meta[\"transcript\"], token_metas):\n", " text = segment_info[\"text\"]\n", " speaker = _norm_speaker_name(segment_info[\"speaker\"])\n", " # split off laughter as a None speaker segment\n", " token_buffer = []\n", " for token, token_timing in token_meta:\n", " token_info = {\"text\": token, \"start_s\": token_timing[0], \"end_s\": token_timing[1]}\n", " if token == \"[laughter]\":\n", " if len(token_buffer) > 0:\n", " segments.append({\n", " \"text\": \" \".join([e[\"text\"] for e in token_buffer]),\n", " \"speaker\": speaker,\n", " \"tokens\": token_buffer,\n", " })\n", " segments.append({\n", " \"text\": token,\n", " \"speaker\": None,\n", " \"tokens\": [token_info],\n", " })\n", " token_buffer = []\n", " else:\n", " token_buffer.append(token_info)\n", " if len(token_buffer) > 0:\n", " segments.append({\n", " \"text\": \" \".join([e[\"text\"] for e in token_buffer]),\n", " \"speaker\": speaker,\n", " \"tokens\": token_buffer,\n", " })\n", " # prepare normalized segments\n", " segments_normed = []\n", " for (\n", " segment_info, text, token_meta\n", " ) in zip(base_meta[\"transcript\"], texts_normed, normed_token_metas):\n", " speaker = _norm_speaker_name(segment_info[\"speaker\"])\n", " tokens = [{\"text\": e[0], \"start_s\": e[1][0], \"end_s\": e[1][1]} for e in token_meta]\n", " segments_normed.append({\n", " \"text\": text,\n", " \"speaker\": speaker,\n", " \"tokens\": tokens,\n", " })\n", " # assemble into final slice meta\n", " slice_meta = {\n", " \"case_uid\": base_meta[\"uuid\"],\n", " \"case_offset_s\": base_meta[\"timestamp\"][0],\n", " \"uid\": \"__\".join(base_meta[\"path\"].split(\".\")[0].split(\"/\")[-2:]),\n", " \"duration_s\": duration_s,\n", " \"filepath\": base_meta[\"path\"],\n", " \"text\": \" \".join([e[\"text\"] for e in base_meta[\"transcript\"]]),\n", " \"text_normalized\": \" \".join(texts_normed),\n", " \"segments\": segments,\n", " \"segments_normalized\": segments_normed,\n", " }\n", " slice_meta_list.append(slice_meta)\n", " n_slice += 1\n", " \n", "print(round(np.sum([e[\"duration_s\"] for e in slice_meta_list]) / 60 / 60, 1), \"hours\")\n", "# 519.9 hours" ] }, { "cell_type": "code", "execution_count": null, "id": "34c427fc", "metadata": {}, "outputs": [], "source": [ "# collect case and speaker metadata\n", "speakers = []\n", "for e in slice_meta_list:\n", " for ee in e[\"segments\"]:\n", " speaker = ee[\"speaker\"]\n", " if speaker is not None:\n", " speakers.append(speaker)\n", "speakers = set(speakers)\n", "case_uids = set([e[\"case_uid\"] for e in slice_meta_list])\n", "\n", "speaker_meta_list = []\n", "for speaker in speakers:\n", " if speaker in speaker_meta_map.index:\n", " reported_gender = speaker_meta_map.loc[speaker][\"reported_gender\"]\n", " elif speaker[:3] == \"Mr.\":\n", " reported_gender = \"male\"\n", " elif speaker[:3] == \"Ms.\":\n", " reported_gender = \"female\"\n", " else:\n", " reported_gender = None\n", " speaker_meta_list.append({\n", " \"speaker\": speaker,\n", " \"reported_gender\": reported_gender,\n", " })\n", " \n", "case_meta_list = []\n", "for case_uid in case_uids:\n", " case_meta_list.append({\n", " \"uid\": case_uid,\n", " \"date\": case_date_map[case_uid],\n", " \"title\": case_title_map[case_uid],\n", " })" ] }, { "cell_type": "code", "execution_count": null, "id": "929cbab0", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"manifest.jsonl\", \"w\") as f:\n", " for e in slice_meta_list:\n", " f.write(json.dumps(e) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "id": "66a00b57", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"speaker_meta.jsonl\", \"w\") as f:\n", " for e in speaker_meta_list:\n", " f.write(json.dumps(e) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "id": "723bab70", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"case_meta.jsonl\", \"w\") as f:\n", " for e in case_meta_list:\n", " f.write(json.dumps(e) + \"\\n\")" ] }, { "cell_type": "markdown", "id": "e46f6775", "metadata": {}, "source": [ "## make test samples" ] }, { "cell_type": "code", "execution_count": null, "id": "21f16bc1", "metadata": {}, "outputs": [], "source": [ "SAMPLE_UIDS = {\n", " \"hesitations\": [\n", " \"21-147__956190_970800\",\n", " \"21-147__1282680_1297650\",\n", " \"21-147__3386510_3396020\",\n", " \"20-1641__4687690_4698920\",\n", " \"20-843__5104170_5115150\",\n", " \"20-443__4623000_4634370\",\n", " ],\n", " \"numbers\": [\n", " \"20-1641__1863190_1878080\",\n", " \"20-493__3600900_3614610\",\n", " \"20-1650__1226900_1241650\",\n", " \"20-1800__593030_604730\",\n", " \"20-1263__2068170_2081989\",\n", " \"20-979__975800_989340\",\n", " ],\n", " \"crosstalk\": [\n", " \"21-147__682320_694210\",\n", " \"20-1410__413460_427000\",\n", " \"20-1410__2062159_2077050\",\n", " ],\n", " \"laughter\": [\n", " \"21-147__3791560_3803480\",\n", " \"20-1530__6494590_6506210\",\n", " \"20-1775__3869650_3879200\",\n", " \"20-493__1300720_1314760\",\n", " \"20-1459__1665940_1677300\",\n", " ],\n", " \"covid\": [\n", " \"21A244__274920_289550\",\n", " \"21A244__2153710_2168030\",\n", " \"21A240__73920_88120\",\n", " ],\n", " \"fun\": [\n", " \"11-1327__1768460_1778230\",\n", " \"18-877__1723210_1736210\",\n", " \"19-1392__1117520_1129990\",\n", " \"20-157__2524330_2533810\",\n", " \"20-440__3911930_3920700\",\n", " ],\n", " \"medical\": [\n", " \"10-945__1492830_1505590\",\n", " \"14-7955__259100_273940\",\n", " \"20-1410__5102270_5113200\",\n", " \"15-7__1922690_1937290\",\n", " ],\n", " \"corporate\": [\n", " \"17-204__2970390_2984880\",\n", " \"13-461__3056820_3066520\",\n", " \"18-556__2219780_2224420\",\n", " ],\n", " \"phone\": [\n", " \"20-440__392330_404830\",\n", " ]\n", "}" ] }, { "cell_type": "code", "execution_count": null, "id": "d147496c", "metadata": {}, "outputs": [], "source": [ "meta_map = {e[\"uid\"]: e for e in slice_meta_list}\n", "\n", "S3_BUCKET_DIR = \"s3://suno-static-public/datasets/supreme-corpus/sample-data/\"\n", "\n", "sample_slice_meta_dict = {}\n", "for k, v in SAMPLE_UIDS.items():\n", " sample_slice_meta_dict[k] = []\n", " for vv in v:\n", " if vv not in meta_map:\n", " print(\"missing:\", vv)\n", " continue\n", " # change to absolute s3 filepath, and add date\n", " meta = meta_map[vv].copy()\n", " meta[\"case_date\"] = case_date_map[meta[\"case_uid\"]]\n", " meta[\"s3_filepath\"] = S3_BUCKET_DIR + \"slices/\" + meta[\"uid\"] + \".wav\"\n", " sample_slice_meta_dict[k].append(meta)\n", " \n", "sample_slice_meta_list = []\n", "for k, v in SAMPLE_UIDS.items():\n", " for vv in v:\n", " if vv not in meta_map:\n", " print(\"missing:\", vv)\n", " continue\n", " sample_slice_meta_list.append(meta_map[vv])" ] }, { "cell_type": "code", "execution_count": null, "id": "d6667486", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"sample_manifest_web.json\", \"w\") as f:\n", " json.dump(sample_slice_meta_dict, f)\n", " \n", "with open(DATA_DIR + \"sample_manifest.jsonl\", \"w\") as f:\n", " for e in sample_slice_meta_list:\n", " f.write(json.dumps(e) + \"\\n\")" ] }, { "cell_type": "markdown", "id": "b5d2ccbb", "metadata": {}, "source": [ "## Create TAR GZ files" ] }, { "cell_type": "code", "execution_count": null, "id": "a0efe383", "metadata": {}, "outputs": [], "source": [ "BASE_DIR = \"/mnt/data-ssd-1/data/supreme_corpus/\"\n", "TMP_FULL_DIR = \"/mnt/data-ssd-1/tmp/full_sc/\"\n", "TMP_SAMPLE_DIR = \"/mnt/data-ssd-1/tmp/sample_sc/\"" ] }, { "cell_type": "code", "execution_count": null, "id": "20586e82", "metadata": {}, "outputs": [], "source": [ "import shutil\n", "\n", "if not os.path.exists(TMP_FULL_DIR):\n", " os.makedirs(TMP_FULL_DIR + \"supreme_corpus/\")\n", " os.makedirs(TMP_SAMPLE_DIR + \"supreme_corpus/\")\n", " shutil.copy(DATA_DIR + \"manifest.jsonl\", TMP_FULL_DIR + \"supreme_corpus/manifest.jsonl\")\n", " shutil.copy(DATA_DIR + \"sample_manifest.jsonl\", TMP_SAMPLE_DIR + \"supreme_corpus/manifest.jsonl\")\n", " shutil.copy(DATA_DIR + \"speaker_meta.jsonl\", TMP_FULL_DIR + \"supreme_corpus/speaker_meta.jsonl\")\n", " shutil.copy(DATA_DIR + \"speaker_meta.jsonl\", TMP_SAMPLE_DIR + \"supreme_corpus/speaker_meta.jsonl\")\n", " shutil.copy(DATA_DIR + \"case_meta.jsonl\", TMP_FULL_DIR + \"supreme_corpus/case_meta.jsonl\")\n", " shutil.copy(DATA_DIR + \"case_meta.jsonl\", TMP_SAMPLE_DIR + \"supreme_corpus/case_meta.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "9f848855", "metadata": {}, "outputs": [], "source": [ "# copy correct slices\n", "sample_from_to_list = []\n", "for e in sample_slice_meta_list:\n", " from_fp = DATA_DIR + e[\"filepath\"]\n", " to_fp = TMP_SAMPLE_DIR + \"supreme_corpus/\" + e[\"filepath\"]\n", " to_dir = \"/\".join(to_fp.split(\"/\")[:-1])\n", " if not os.path.exists(to_dir):\n", " os.makedirs(to_dir)\n", " sample_from_to_list.append((from_fp, to_fp))\n", " \n", "full_from_to_list = []\n", "for e in slice_meta_list:\n", " from_fp = DATA_DIR + e[\"filepath\"]\n", " to_fp = TMP_FULL_DIR + \"supreme_corpus/\" + e[\"filepath\"]\n", " to_dir = \"/\".join(to_fp.split(\"/\")[:-1])\n", " if not os.path.exists(to_dir):\n", " os.makedirs(to_dir)\n", " full_from_to_list.append((from_fp, to_fp))\n", " \n", "def do_copy(combo):\n", " from_fp, to_fp = combo\n", " shutil.copy(from_fp, to_fp)\n", " \n", "p = Pool(32)\n", "p.map(do_copy, sample_from_to_list)\n", "p.map(do_copy, full_from_to_list)\n", "p.close()\n", "p.join()" ] }, { "cell_type": "code", "execution_count": null, "id": "dc38b486", "metadata": {}, "outputs": [], "source": [ "## tag gz and move to main dir\n", "\n", "# cd /mnt/data-ssd-1/tmp/sample_sc && \\\n", "# tar -c supreme_corpus | \\\n", "# pigz -5 -p 32 > /mnt/data-ssd-1/data/supreme_corpus/supreme_corpus_sample.tar.gz && \\\n", "# cd -\n", "\n", "# cd /mnt/data-ssd-1/tmp/full_sc && \\\n", "# tar -c supreme_corpus | \\\n", "# pigz -5 -p 32 > /mnt/data-ssd-1/data/supreme_corpus/supreme_corpus.tar.gz && \\\n", "# cd -\n", "\n", "## 519.9 hours, 155gb uncompressed -> 132gb compressed" ] }, { "cell_type": "code", "execution_count": null, "id": "edbac277", "metadata": {}, "outputs": [], "source": [ "# cleanup\n", "shutil.rmtree(TMP_FULL_DIR)\n", "shutil.rmtree(TMP_SAMPLE_DIR)" ] }, { "cell_type": "markdown", "id": "ea54a2cd", "metadata": {}, "source": [ "### upload static website data" ] }, { "cell_type": "code", "execution_count": null, "id": "f38205b9", "metadata": {}, "outputs": [], "source": [ "# s3cmd put \\\n", "# /mnt/data-ssd-1/data/supreme_corpus/processed_data/sample_manifest_web.json \\\n", "# s3://suno-static-public/datasets/supreme-corpus/sample-data/" ] }, { "cell_type": "code", "execution_count": null, "id": "9dc0e8f6", "metadata": {}, "outputs": [], "source": [ "# s3ptn = \"s3cmd put {} {}\"\n", "# cmd_list = []\n", "# for k, v in sample_slice_meta_dict.items():\n", "# for e in v:\n", "# from_fp = DATA_DIR + e[\"filepath\"]\n", "# cmd_list.append(s3ptn.format(from_fp, e[\"s3_filepath\"]))\n", "# print(\" && \".join(cmd_list))" ] }, { "cell_type": "code", "execution_count": null, "id": "2563674e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9d64a53a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "31088388", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7aa9a72b", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "markdown", "id": "2301f9bc", "metadata": {}, "source": [ "### Opus conversion" ] }, { "cell_type": "code", "execution_count": null, "id": "bc7a0701", "metadata": {}, "outputs": [], "source": [ "# LD_LIBRARY_PATH=/usr/local/lib /usr/local/bin/opusenc \\\n", "# --bitrate 64 \\\n", "# --speech \\\n", "# /mnt/data-ssd-1/data/supreme_corpus/processed_data/slices/11-1059/3350270_3365270.wav \\\n", "# /home/georg/notebooks/datasets/supreme_court/test.opus" ] }, { "cell_type": "markdown", "id": "97d2bb0f", "metadata": {}, "source": [ "### Example output" ] }, { "cell_type": "code", "execution_count": null, "id": "6340babc", "metadata": {}, "outputs": [], "source": [ "# {\n", "# 'case_uid': '99-999',\n", "# 'case_offset_s': 3.0,\n", "# 'uid': '99-999__3000_8000',\n", "# 'duration_s': 5.0,\n", "# 'filepath': 'slices/99-999/0_5000.wav',\n", "# 'text': 'Hi there. [laughter] Good morning.',\n", "# 'text_normalized': 'hi there good morning',\n", "# 'segments': [\n", "# {\n", "# 'text': 'Hi there.',\n", "# 'speaker': 'Mr. XXX',\n", "# 'tokens': [\n", "# {'text': 'Hi', 'start_s': 0.5, 'end_s': 1.0},\n", "# {'text': 'there.', 'start_s': 1.5, 'end_s': 2.0},\n", "# ],\n", "# }, {\n", "# 'text': '[laughter]',\n", "# 'speaker': null,\n", "# 'tokens': [\n", "# {'text': '[laughter]', 'start_s': 2.0, 'end_s': 2.5},\n", "# ],\n", "# }, {\n", "# 'text': 'Good morning.',\n", "# 'speaker': 'Ms. YYY',\n", "# 'tokens': [\n", "# {'text': 'Good', 'start_s': 2.5, 'end_s': 3.0},\n", "# {'text': 'morning.', 'start_s': 3.5, 'end_s': 4.0},\n", "# ],\n", "# },\n", "# ],\n", "# 'segments_normalized': [\n", "# {\n", "# 'text': 'hi there',\n", "# 'speaker': 'Mr. XXX',\n", "# 'tokens': [\n", "# {'text': 'hi', 'start_s': 0.5, 'end_s': 1.0},\n", "# {'text': 'there', 'start_s': 1.5, 'end_s': 2.0},\n", "# ],\n", "# }, {\n", "# 'text': 'good morning',\n", "# 'speaker': 'Ms. YYY',\n", "# 'tokens': [\n", "# {'text': 'good', 'start_s': 2.5, 'end_s': 3.0},\n", "# {'text': 'morning', 'start_s': 3.5, 'end_s': 4.0},\n", "# ],\n", "# },\n", "# ],\n", "# }" ] }, { "cell_type": "code", "execution_count": null, "id": "f60f9467", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ee5aa7cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0e916a90", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "668f3199", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "70d2aade", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "653d654b", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }