{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "64d773c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fe6f477a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0d497284",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import re\n",
    "import shutil\n",
    "\n",
    "from multiprocessing import Pool\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pydub import AudioSegment\n",
    "import sox\n",
    "import tqdm\n",
    "\n",
    "from suno_utils.utils.notebook import play_audio, suppress_logging\n",
    "from suno_utils.utils.text import normalize_whitespace\n",
    "from suno_utils.utils.conversion import convert_audio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "188ba833",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # main corpus stats\n",
    "# ~4k hours\n",
    "# 8.0 bit depth\n",
    "# 11025 sample rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "55eb1c96",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAIN_DIR = \"/mnt/data-ssd-1/data/georgia_court/\"\n",
    "DRIVE_DATA_DIR = MAIN_DIR + \"raw_drive_data/\"\n",
    "DATA_DIR = MAIN_DIR + \"raw_data/\"\n",
    "\n",
    "SAMPLE_RATE = 16_000\n",
    "BYTE_WIDTH = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "592c2853",
   "metadata": {},
   "source": [
    "## Convert super raw data to raw data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "f9346c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def _check_is_valid(fp):\n",
    "    with suppress_logging():\n",
    "        duration_s = sox.file_info.duration(fp)\n",
    "    if duration_s is None:\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "8e684765",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 ambiguous txt files dropped\n",
      "0 ambiguous wav files dropped\n",
      "11 corrupt wav files dropped\n",
      "1553 valid pairs found\n"
     ]
    }
   ],
   "source": [
    "# find matching pairs\n",
    "wav_file_list = []\n",
    "txt_file_list = []\n",
    "for dir_name in os.listdir(DRIVE_DATA_DIR):\n",
    "    from_dir_path = os.path.join(DRIVE_DATA_DIR, dir_name)\n",
    "    to_dir_path = os.path.join(DATA_DIR, dir_name.lower().replace(\" \", \"_\"))\n",
    "    if not os.path.exists(to_dir_path):\n",
    "        os.makedirs(to_dir_path)\n",
    "    for file_name in os.listdir(from_dir_path):\n",
    "        from_file_path = os.path.join(from_dir_path, file_name)\n",
    "        to_file_path = os.path.join(to_dir_path, file_name)\n",
    "        if file_name[-4:] == \".wav\":\n",
    "            wav_file_list.append((file_name[:-4], os.path.getsize(from_file_path), from_file_path, to_file_path))\n",
    "        elif file_name[-4:] == \".txt\":\n",
    "            txt_file_list.append((file_name[:-4], os.path.getsize(from_file_path), from_file_path, to_file_path))\n",
    "\n",
    "# check for remove ambiguous ones\n",
    "txt_df = pd.DataFrame(txt_file_list, columns=[\"uid\", \"file_size\", \"from_fp\", \"to_fp\"])\n",
    "txt_df = txt_df.drop_duplicates(subset=[\"uid\", \"file_size\"], keep=\"first\")\n",
    "drop_idxs = (txt_df[\"uid\"].value_counts() > 1).loc[lambda x: x].index.tolist()\n",
    "txt_df = txt_df[~txt_df[\"uid\"].isin(drop_idxs)]\n",
    "print(len(drop_idxs), \"ambiguous txt files dropped\")\n",
    "wav_df = pd.DataFrame(wav_file_list, columns=[\"uid\", \"file_size\", \"from_fp\", \"to_fp\"])\n",
    "wav_df = wav_df.drop_duplicates(subset=[\"uid\", \"file_size\"], keep=\"first\").reset_index(drop=True)\n",
    "drop_idxs = (wav_df[\"uid\"].value_counts() > 1).loc[lambda x: x].index.tolist()\n",
    "wav_df = wav_df[~wav_df[\"uid\"].isin(drop_idxs)]\n",
    "print(len(drop_idxs), \"ambiguous wav files dropped\")\n",
    "\n",
    "# check for corrupt audio (forgot to turn of)\n",
    "corrupt_uids = set([row[\"uid\"] for _, row in wav_df.iterrows() if not _check_is_valid(row[\"from_fp\"])])\n",
    "wav_df = wav_df[~wav_df[\"uid\"].isin(corrupt_uids)]\n",
    "print(len(corrupt_uids), \"corrupt wav files dropped\")\n",
    "\n",
    "# check for valid pairs\n",
    "valid_uids = set(wav_df[\"uid\"]) & set(txt_df[\"uid\"])\n",
    "wav_df = wav_df[wav_df[\"uid\"].isin(valid_uids)]\n",
    "txt_df = txt_df[txt_df[\"uid\"].isin(valid_uids)]\n",
    "print(len(valid_uids), \"valid pairs found\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "7ccfe513",
   "metadata": {},
   "outputs": [],
   "source": [
    "# move non-ambiguous ones to new directory and convert to sane format\n",
    "def _convert_audio(combo):\n",
    "    from_fp, to_fp = combo\n",
    "    if not os.path.exists(to_fp):\n",
    "        convert_audio(from_fp, to_fp, n_channels=1, sample_rate=SAMPLE_RATE, byte_width=BYTE_WIDTH)\n",
    "\n",
    "for _, row in txt_df.iterrows():\n",
    "    if not os.path.exists(row[\"to_fp\"]):\n",
    "        shutil.copy(row[\"from_fp\"], row[\"to_fp\"])\n",
    "\n",
    "combo_list = list(zip(wav_df[\"from_fp\"], wav_df[\"to_fp\"]))\n",
    "\n",
    "p = Pool(32)\n",
    "_ = p.map(_convert_file, combo_list)\n",
    "p.close()\n",
    "p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "219a32c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save metadata file\n",
    "meta_df = pd.merge(\n",
    "    wav_df[[\"uid\", \"to_fp\"]].rename(columns={\"to_fp\": \"wav_filepath\"}),\n",
    "    txt_df[[\"uid\", \"to_fp\"]].rename(columns={\"to_fp\": \"txt_filepath\"}),\n",
    "    on=\"uid\"\n",
    ")\n",
    "assert(meta_df.shape[0] == wav_df.shape[0] == txt_df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "f610486e",
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_df.to_csv(MAIN_DIR + \"raw_meta.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0739fbb5",
   "metadata": {},
   "source": [
    "## check txt court doc parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "deee08d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_df = pd.read_csv(MAIN_DIR + \"raw_meta.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c3c47139",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from suno_utils.datasets.supreme_court.parser import\n",
    "\n",
    "# play_audio(row[\"wav_filepath\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b99023aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "INTERPRETER_LINES = [\n",
    "    \"interpreter, having been first duly sworn\",\n",
    "    \"interpret accurately the testimony\",\n",
    "    \"interpreted the proceedings as follows\",\n",
    "]\n",
    "\n",
    "def _check_for_foreign_language(doc):\n",
    "    if any([s in doc for s in INTERPRETER_LINES]):\n",
    "        raise ValueError(\"foreign language present\")\n",
    "\n",
    "def _get_examinee(lines):\n",
    "    toc_lines = []\n",
    "    for text in lines:\n",
    "        # allow max two lowercase letters\n",
    "        m = re.search(r\"^\\s*[0-9]+\\s+EXAMINATION OF ([^a-z]+?[a-z]?[^a-z]+?[a-z]?[^a-z]+?)\\s\\s+.*$\", text)\n",
    "        if m:\n",
    "            toc_lines.append(m.group(1))\n",
    "    if len(toc_lines) == 0:\n",
    "        raise ValueError(\"no defendant found\")\n",
    "    if len(toc_lines) > 1:\n",
    "        raise ValueError(\"no unique defendant found\")\n",
    "    return toc_lines[0]\n",
    "\n",
    "def _filter_to_valid_lines(lines):\n",
    "    valid_lines = []\n",
    "    for l in lines:\n",
    "        m = re.search(r\"^\\s*[0-9]+\\s+(.*[^\\s]{3,}.*)$\", l)\n",
    "        if m:\n",
    "            valid_lines.append(normalize_whitespace(m.group(1)))\n",
    "    return valid_lines\n",
    "\n",
    "\n",
    "PROCEEDING_END_LINES = [\n",
    "    \"(Proceedings concluded at\",\n",
    "    \"(Proceedings suspended at\",\n",
    "    \"(Proceedings adjourned at\",\n",
    "    \"(Examination concluded at\",\n",
    "]\n",
    "\n",
    "def _get_content_lines(lines):\n",
    "    begin_idx = -1\n",
    "    start_idx = -1\n",
    "    end_idx = -1\n",
    "    for n, text in enumerate(lines):\n",
    "        if re.search(r\"P?\\s+R\\s+O\\s+C\\s+E\\s+E\\s+D\\s+I\\s+N\\s+G\\s+S\", text):\n",
    "            if begin_idx >= 0:\n",
    "                raise ValueError(\"found multiple beginnings.\")\n",
    "            begin_idx = n + 1\n",
    "        if \"EXAMINATION\" in text:\n",
    "            if start_idx < 0 and begin_idx >= 0:\n",
    "                start_idx = n\n",
    "        if any([s in text for s in PROCEEDING_END_LINES]):\n",
    "            if end_idx >= 0:\n",
    "                raise ValueError(\"found multiple end pages.\")\n",
    "            end_idx = n + 1\n",
    "    if begin_idx < 0:\n",
    "        raise ValueError(\"beginning not found.\")\n",
    "    if start_idx < 0:\n",
    "        raise ValueError(\"start not found.\")\n",
    "    if end_idx < 0:\n",
    "        raise ValueError(\"end not found.\")\n",
    "    if end_idx - start_idx < 50:\n",
    "        raise ValueError(\"too few lines.\")\n",
    "    valid_lines = _filter_to_valid_lines(lines[start_idx:end_idx])\n",
    "    return valid_lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "id": "c7db8467",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1553/1553 [00:41<00:00, 37.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "183/1553 failed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "failed_ids = []\n",
    "for _, row in tqdm.tqdm(meta_df.iterrows(), total=meta_df.shape[0]):\n",
    "    with open(row[\"txt_filepath\"], encoding=\"Windows-1252\") as f:\n",
    "        doc = f.read()\n",
    "    try:\n",
    "        _check_for_foreign_language(doc)\n",
    "        raw_lines = doc.split(\"\\n\")\n",
    "        examinee_name = _get_examinee(raw_lines)\n",
    "        content_lines = _get_content_lines(raw_lines)\n",
    "    except Exception as e:\n",
    "        failed_ids.append((row[\"uid\"], e, len(doc)))\n",
    "\n",
    "print(\"{}/{} failed\".format(len(failed_ids), meta_df.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "id": "e71e1440",
   "metadata": {},
   "outputs": [],
   "source": [
    "## TODO: 'no defendant found' often just different format\n",
    "## TODO: 'foreign language present' might still be useable\n",
    "\n",
    "## analyze failures\n",
    "# df = pd.DataFrame(failed_ids, columns=[\"uid\", \"error\", \"doc_len\"])\n",
    "# df[\"error\"] = df[\"error\"].astype(str)\n",
    "# s = df.groupby(\"error\")[\"uid\"].count().sort_values(ascending=False)\n",
    "# for e in s.index:\n",
    "#     print(\"({}) {}\".format(s.loc[e], e))\n",
    "#     print(df[df[\"error\"] == e][\"uid\"].tolist()[:10])\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55a6e134",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: get type of hearing??\n",
    "\n",
    "# CIVIL ACTION FILE\n",
    "# CASE NO.\n",
    "# Fair Dismissal Act\n",
    "# Hearing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d69942a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: check for other headers, eg:\n",
    "\n",
    "# '061813BCTG'\n",
    "# COMPANY'S OPENING STATEMENT:\n",
    "# UNION'S OPENING STATEMENT:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43a214ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: parse sections and corresponding speakers\n",
    "\n",
    "# EXAMINATION\n",
    "# BY MR. HATTON:\n",
    "# A. 1987.\n",
    "# A. E4.\n",
    "# 1994.\n",
    "# BI-LO.\n",
    "# A. 1994.\n",
    "# A. 2003.\n",
    "# 5000?\n",
    "# EXAMINATION\n",
    "# BY MR. HINTON:\n",
    "# FURTHER EXAMINATION\n",
    "# BY MR. HATTON:\n",
    "# FURTHER EXAMINATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "def8e3d5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0a04147",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b15e9020",
   "metadata": {},
   "source": [
    "## check type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3e086208",
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_df = pd.read_csv(MAIN_DIR + \"raw_meta.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1e79f05b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1553it [00:00, 3567.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "629 known\n",
      "924 unknown\n",
      "0 failed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "known_uids = []\n",
    "failed_uids = []\n",
    "unknown_uids = []\n",
    "for n, (_, row) in tqdm.tqdm(enumerate(meta_df.iterrows())):\n",
    "    with open(row[\"txt_filepath\"], encoding=\"Windows-1252\") as f:\n",
    "        doc = f.read()[:2000]\n",
    "        \n",
    "    is_known = False\n",
    "#     if (\n",
    "#         \"CIVIL ACTION FILE\" in doc or \n",
    "#         \"CIVIL ACTION\" in doc# or\n",
    "# #         (\"plaintiff\" in doc.lower() and \"defendant\" in doc.lower())\n",
    "#     ):\n",
    "#         if is_known:\n",
    "#             failed_uids.append(row[\"uid\"])\n",
    "#             continue\n",
    "#         is_known = True\n",
    "    if \"WORKERS' COMPENSATION\" in doc:# or \"claim no.\" in doc.lower() or \"policy no.\" in doc.lower():\n",
    "        if is_known:\n",
    "            failed_uids.append(row[\"uid\"])\n",
    "            continue\n",
    "        is_known = True\n",
    "    if \"BANKRUPTCY COURT\" in doc:\n",
    "        if is_known:\n",
    "            failed_uids.append(row[\"uid\"])\n",
    "            continue\n",
    "        is_known = True\n",
    "#     if \"injury\" in doc.lower() and \"WORKERS\" not in doc:\n",
    "#         if is_known:\n",
    "#             print(doc)\n",
    "#             break\n",
    "#             failed_uids.append(row[\"uid\"])\n",
    "#             continue\n",
    "#         is_known = True\n",
    "#     if \"statement\" in doc.lower():  # eg sworn statement under oath\n",
    "#         if is_known:\n",
    "#             failed_uids.append(row[\"uid\"])\n",
    "#             continue\n",
    "#         is_known = True\n",
    "#     if \"LITIGATION\" in doc:\n",
    "#         if is_known:\n",
    "#             failed_uids.append(row[\"uid\"])\n",
    "#             continue\n",
    "#         is_known = True\n",
    "    if is_known:\n",
    "        known_uids.append(row[\"uid\"])\n",
    "    else:\n",
    "        unknown_uids.append(row[\"uid\"])\n",
    "        \n",
    "print(len(known_uids), \"known\")\n",
    "print(len(unknown_uids), \"unknown\")\n",
    "print(len(failed_uids), \"failed\")\n",
    "# 637 workers' comp\n",
    "#   9 bankruptcy\n",
    "# 907 unknown\n",
    "# 0 failed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27f23935",
   "metadata": {},
   "outputs": [],
   "source": [
    "# deposition in a law firm, goverened by court of jurisdiction\n",
    "# need to check for confidential\n",
    "# big 3: workers' comp, Personal Injury and Medical Malpractice\n",
    "# \"examination under oath\" usually indicates that it's an insurance loss case rather than a lawsuit\n",
    "# jurisdiction / venue. I.e., the jurisdiction (state, superior court, federal, board of workers comp, etc.) where the case is being filed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 531,
   "id": "4d92802f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !rm tmp/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "957d4f3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20it [00:00, 1196.54it/s]\n"
     ]
    }
   ],
   "source": [
    "for n, uid in tqdm.tqdm(enumerate(unknown_uids[:20])):\n",
    "    row = meta_df[meta_df[\"uid\"] == uid].iloc[0]\n",
    "    with open(row[\"txt_filepath\"], encoding=\"Windows-1252\") as f:\n",
    "        doc = f.read()\n",
    "    with open(\"tmp/{}.txt\".format(n), \"w\") as f:\n",
    "        f.write(doc[:5000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 533,
   "id": "6a3718aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# uid = unknown_uids[43]\n",
    "# row = meta_df[meta_df[\"uid\"] == uid].iloc[0]\n",
    "# with open(row[\"txt_filepath\"], encoding=\"Windows-1252\") as f:\n",
    "#     doc = f.read()\n",
    "# print(doc[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78ccf4f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75d42f5b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f41eb62",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "985d4aec",
   "metadata": {},
   "source": [
    "## Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ede2bd78",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import nemo.collections.asr as nemo_asr\n",
    "# asr_model = nemo_asr.models.EncDecRNNTBPEModel.from_pretrained(model_name=\"stt_en_contextnet_1024\")\n",
    "# asr_model.transcribe([\"/mnt/data-ssd-1/data/court/042712MCKE_mini.wav\"])[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd609ef2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(\"/mnt/data-ssd-1/data/court/042712MCKE_raw.txt\") as f:\n",
    "#     doc = f.read()\n",
    "\n",
    "# def whitespace_normalize(s):\n",
    "#     return re.sub(r\"\\s+\", \" \", s).strip()\n",
    "\n",
    "# lines = [whitespace_normalize(l) for l in doc.split(\"\\n\")]\n",
    "\n",
    "# for n, l in enumerate(lines):\n",
    "#     if \"P R O C E E D I N G S\" in l:\n",
    "#         lines = lines[n+1:]\n",
    "#         print(\"found initializer!\")\n",
    "#         break\n",
    "\n",
    "# for n, l in enumerate(lines):\n",
    "#     if \"EXAMINATION\" in l:\n",
    "#         lines = lines[n+1:]\n",
    "#         print(\"found start!\")\n",
    "#         break\n",
    "        \n",
    "# for n, l in enumerate(lines):\n",
    "#     if \"Proceedings concluded at\" in l:\n",
    "#         lines = lines[:n]\n",
    "#         print(\"found end!\")\n",
    "#         break\n",
    "        \n",
    "# convo_lines = []\n",
    "# rem_lines = []\n",
    "# for l in lines:\n",
    "#     if len(l) == 0:\n",
    "#         continue\n",
    "#     m = re.search(r\"[0-9]+\\s{1,10}(.*[a-z0-9].*)\", l)\n",
    "#     if m:\n",
    "#         p1 = r\"^[0-9]+\\s+Q\\.\\s+\"\n",
    "#         p2 = r\"^[0-9]+\\s+A\\.\\s+\"\n",
    "#         p3 = r\"^[0-9]+\\s+[A-Z\\. ]+\\:\\s+\"\n",
    "#         p4 = r\"^[0-9]+\\s+\"\n",
    "#         if re.search(p1, l):\n",
    "#             l = re.sub(p1, \"\", l)\n",
    "#         elif re.search(p2, l):\n",
    "#             l = re.sub(p2, \"\", l)\n",
    "#         elif re.search(p3, l):\n",
    "#             l = re.sub(p3, \"\", l)\n",
    "#         elif re.search(p4, l):\n",
    "#             l = re.sub(p4, \"\", l)\n",
    "#         convo_lines.append(l)\n",
    "#     else:\n",
    "#         rem_lines.append(l)\n",
    "# text = \" \".join(convo_lines)\n",
    "# print(\"merged text!\")\n",
    "\n",
    "# assert(all([len(m) <= 200 for m in re.findall(r\"\\(.+?\\)\", text)]))\n",
    "# text = re.sub(r\"\\(.+?\\)\", \" \", text)\n",
    "# text = whitespace_normalize(text)\n",
    "# print(\"cleaned text!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e017c25",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12c94456",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62a3b37e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e1cb167",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3a6699a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b4afbb0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}