{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "90179a9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tqdm\n",
    "import re\n",
    "import os\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "185541c0",
   "metadata": {},
   "source": [
    "## Extract speech papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e6ff0220",
   "metadata": {},
   "outputs": [],
   "source": [
    "ml_papers_df = pd.read_json(\n",
    "    \"/mnt/data-ssd-2/data/arxiv/arxiv_ml_meta.json\", \n",
    "    dtype={\"id\": str},\n",
    "    lines=True,\n",
    ")\n",
    "ml_papers_df[\"submit_date\"] = pd.to_datetime(ml_papers_df[\"submit_date\"], unit=\"ms\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a3849e28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "3856 total papers found\n"
     ]
    }
   ],
   "source": [
    "min_year = 2018\n",
    "TASK_SEARCH_PTNS = {\n",
    "    \"ASR\": [\n",
    "        r\"\\b[Ss]peech [Rr]ecognition\\b\",\n",
    "        r\"\\bASR\\b\",\n",
    "        r\"\\bSTT\\b\",\n",
    "    ],\n",
    "    \"TTS\": [\n",
    "        r\"\\b[Tt]ext [Tt]o [Ss]peech\\b\",\n",
    "        r\"\\b[Ss]peech [Ss]ynthesis\\b\",\n",
    "        r\"\\bTTS\\b\",\n",
    "    ],\n",
    "    \"Diarization\": [\n",
    "        r\"\\b[Dd]iarization\\b\",\n",
    "        r\"\\b[Ss]peaker [Ee]mbedding\\b\",\n",
    "        r\"\\b[Ss]peaker [Vv]erification\\b\",\n",
    "    ],\n",
    "    \"Speech Enh.\": [\n",
    "        r\"\\b[Ss]peech [Ee]nhancement\\b\",\n",
    "        r\"\\b[Ss]peech.+[Dd]enois\",\n",
    "        r\"\\b[Dd]enois.+[Ss]peech\\b\",\n",
    "    ],\n",
    "    \"Misc\": [\n",
    "        r\"\\b[Ss]peech [Rr]epresentation\\b\",\n",
    "        r\"\\b[Ss]peech [Pp]processing\\b\",\n",
    "        r\"\\b[Ii][Nn][Tt][Ee][Rr][Ss][Pp][Ee][Ee][Cc][Hh]\\b\",\n",
    "        r\"\\b[Ii][Cc][Aa][Ss][Ss][Pp]\\b\",\n",
    "    ],\n",
    "}\n",
    "\n",
    "# get ids for all tasks\n",
    "df = ml_papers_df.copy()\n",
    "df = df[df[\"submit_date\"].dt.year >= min_year]\n",
    "df[\"haystack\"] = df[\"title\"] + \" \" + df[\"abstract\"]\n",
    "all_paper_ids = set()\n",
    "task_to_paper_ids = {}\n",
    "for task, search_ptns in TASK_SEARCH_PTNS.items():\n",
    "    search_ptn = r\"(?:\" + r\"|\".join(search_ptns) + r\")\"\n",
    "    paper_ids = set(df[df[\"haystack\"].str.contains(search_ptn, regex=True, flags=re.DOTALL)][\"id\"])\n",
    "    if task == \"Misc\":\n",
    "        paper_ids -= all_paper_ids\n",
    "    task_to_paper_ids[task] = paper_ids\n",
    "    all_paper_ids |= paper_ids\n",
    "    ps = str(len(paper_ids))\n",
    "    ps = \" \" * (18 - len(task) - len(ps)) + ps\n",
    "    print(f\"{task}: {ps} papers found\")\n",
    "\n",
    "paper_id_to_tasks = defaultdict(list)\n",
    "for task, paper_ids in task_to_paper_ids.items():\n",
    "    for paper_id in paper_ids:\n",
    "        paper_id_to_tasks[paper_id].append(task)\n",
    "    \n",
    "all_paper_ids = set()\n",
    "for _, v in task_to_paper_ids.items():\n",
    "    all_paper_ids |= v\n",
    "print(f\"\\n{len(all_paper_ids)} total papers found\")\n",
    "\n",
    "# ASR:            2910 papers found\n",
    "# TTS:             691 papers found\n",
    "# Diarization:     684 papers found\n",
    "# Speech Enh.:     553 papers found\n",
    "# Misc:             82 papers found\n",
    "\n",
    "# 4527 total papers found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39849f4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "known_ids = [\n",
    "    \"1910.06711\",  # lyrebird\n",
    "    \"2104.11348\",  # rev\n",
    "    \"2110.13900\",  # wavlm\n",
    "    \"2101.01902\",  # dns challenge\n",
    "    \"2104.02014\",  # spgispeech\n",
    "    \"2106.06909\",  # gigaspeech\n",
    "    \"1804.00015\",  # espnet\n",
    "]\n",
    "assert(len(set(known_ids) - all_paper_ids) == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "56ff654f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"haystack\"].str.extract(r\"(.{,10}[Nn]oise [Ss]uppression.{,10})\").dropna()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0df75417",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9182f96f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "be3093c6",
   "metadata": {},
   "source": [
    "## Set up Custom Extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6647383c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3856/3856 [00:06<00:00, 552.61it/s]\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "FULLTEXT_DIR = \"/mnt/data-ssd-2/data/arxiv/fulltext/arxiv/\"\n",
    "\n",
    "# def extract_domains_from_fulltext(filepath):\n",
    "#     if not os.path.exists(filepath):\n",
    "#         return []\n",
    "#     with open(filepath) as f:\n",
    "#         fulltext = f.read()\n",
    "#     domains = re.findall(r\"\\@([^\\@\\s\\,\\;\\>\\)\\]\\}]+)\", fulltext.lower())\n",
    "#     domains = [re.sub(r\"[^a-z]*$\", \"\", s) for s in domains]\n",
    "#     domains = list(set(domains))\n",
    "#     return domains\n",
    "\n",
    "def extract_emails_from_fulltext(filepath):\n",
    "    if not os.path.exists(filepath):\n",
    "        return []\n",
    "    with open(filepath) as f:\n",
    "        fulltext = f.read()\n",
    "    emails = re.findall(r\"[\\w\\.-]+@[\\w\\.-]+\\.\\w+\", fulltext.lower())\n",
    "    return emails\n",
    "\n",
    "paper_id_to_emails = {}\n",
    "for paper_id in tqdm.tqdm(all_paper_ids):\n",
    "    paper_dir = paper_id.split(\".\")[0]\n",
    "    filepath = FULLTEXT_DIR + f\"{paper_dir}/{paper_id}.txt\"\n",
    "    emails = extract_emails_from_fulltext(filepath)\n",
    "    if len(emails) > 0:\n",
    "        paper_id_to_emails[paper_id] = emails\n",
    "\n",
    "emails_to_paper_ids = defaultdict(list)\n",
    "for paper_id, emails in paper_id_to_emails.items():\n",
    "    for email in emails:\n",
    "        emails_to_paper_ids[email].append(paper_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "78fab90c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1910.06711: ['kundan@descript.com', 'rithesh@descript.com']\n",
      "2104.11348: ['miguel.delrio@rev.com']\n",
      "2110.13900: ['yuwu1@microsoft.com']\n",
      "2101.01902: ['chkarada@microsoft.com', 'firstname.lastname@microsoft.com']\n",
      "2104.02014: ['patrick.oneill@kensho.com', 'georg@kensho.com']\n",
      "2106.06909: ['gigaspeech@speechcolab.org']\n",
      "1804.00015: ['shinjiw@jhu.edu']\n",
      "2103.13581: ['rwang@tongji.edu.cn', 'wei@tongji.edu.cn', 'h.duan5@newcastle.ac.uk', 'sji@zju.edu.cn', 'zhong1983@zjut.edu.cn']\n"
     ]
    }
   ],
   "source": [
    "known_ids = [\n",
    "    \"1910.06711\",  # lyrebird\n",
    "    \"2104.11348\",  # rev\n",
    "    \"2110.13900\",  # wavlm\n",
    "    \"2101.01902\",  # dns challenge\n",
    "    \"2104.02014\",  # spgispeech\n",
    "    \"2106.06909\",  # gigaspeech\n",
    "    \"1804.00015\",  # espnet\n",
    "    \"1909.09577\",  # nemo\n",
    "    \"2103.13581\",  # multiple orgs\n",
    "]\n",
    "for paper_id in known_ids:\n",
    "    if paper_id in paper_id_to_emails:\n",
    "        print(\"{}: {}\".format(paper_id, paper_id_to_emails[paper_id]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a16b735f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb839bb4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02a2481e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dca4a7ac",
   "metadata": {},
   "source": [
    "## Make datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "9445ce8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: company sorted by total papers, list tasks and relevant papers\n",
    "#   filter out orgs\n",
    "#   check existence of domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "88e45746",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "def check_url_exists(url):\n",
    "    b_exists = False\n",
    "    try:\n",
    "        response = requests.head(url, timeout=2)\n",
    "        if response.status_code < 400:\n",
    "            b_exists = True\n",
    "    except:\n",
    "        pass\n",
    "    return b_exists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d98b2bc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# google.com: {\n",
    "#     \"asr\": [\n",
    "#         john@google.com: [1111.111]\n",
    "#     ]\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1591bc16",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3004/3004 [12:09<00:00,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154 companies found\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "BLOCKED_DOMAINS = set([\"gmail.com\", \"outlook.com\", \"hotmail.com\"])\n",
    "\n",
    "company_map = {}\n",
    "failed = []\n",
    "for email, paper_ids in tqdm.tqdm(emails_to_paper_ids.items()):\n",
    "    domain = email.split(\"@\")[-1]\n",
    "    if domain in BLOCKED_DOMAINS:\n",
    "        continue\n",
    "    if re.search(r\"\\.(?:edu|org|ac)\\b\", domain):\n",
    "        continue\n",
    "    if not re.search(r\"\\.(?:com|ai|io)\\b\", domain):\n",
    "        continue\n",
    "    if not check_url_exists(\"https://\" + domain):\n",
    "        if not check_url_exists(\"http://\" + domain):\n",
    "            failed.append(domain)\n",
    "            continue\n",
    "    if domain not in company_map:\n",
    "        company_map[domain] = {}\n",
    "    for paper_id in paper_ids:\n",
    "        tasks = paper_id_to_tasks[paper_id]\n",
    "        for task in tasks:\n",
    "            if task not in company_map[domain]:\n",
    "                company_map[domain][task] = defaultdict(list)\n",
    "#             if email not in company_map[domain][task]:\n",
    "#                 company_map[domain][task][email] = defaultdict(list)\n",
    "            company_map[domain][task][email].append(paper_id)\n",
    "print(len(company_map), \"companies found\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "a1ed0dc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "known_companies = [\n",
    "    \"descript.com\",\n",
    "    \"rev.com\",\n",
    "    \"microsoft.com\",\n",
    "    \"kensho.com\",\n",
    "    \"nvidia.com\",\n",
    "    \"seasalt.ai\",\n",
    "]\n",
    "assert(len(set(known_companies) - set(list(company_map.keys()))) == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9e97e65f",
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_id_2_title = ml_papers_df.set_index(\"id\")[\"title\"].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "35d66e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assemble\n",
    "company_data = []\n",
    "for domain, v in company_map.items():\n",
    "    for task, vv in v.items():\n",
    "        for email, vvv in vv.items():\n",
    "            for paper_id in vvv:\n",
    "                paper_title = paper_id_2_title[paper_id]\n",
    "                company_data.append((domain, email, paper_id, paper_title, task))\n",
    "company_df = pd.DataFrame(\n",
    "    company_data,\n",
    "    columns=[\"company_domain\", \"email\", \"paper_id\", \"paper_title\", \"paper_task\"],\n",
    ")\n",
    "# TODO: remove duplicates and fishy emails?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4142696b",
   "metadata": {},
   "outputs": [],
   "source": [
    "company_df.to_csv(\"fullbast_email.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f3fbdd6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_domain</th>\n",
       "      <th>email</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>paper_title</th>\n",
       "      <th>paper_task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>708</th>\n",
       "      <td>reverieinc.com</td>\n",
       "      <td>shakti.rath@reverieinc.com</td>\n",
       "      <td>2112.01023</td>\n",
       "      <td>A higher order Minkowski loss for improved pre...</td>\n",
       "      <td>ASR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>709</th>\n",
       "      <td>reverieinc.com</td>\n",
       "      <td>shakti.rath@reverieinc.com</td>\n",
       "      <td>2112.01025</td>\n",
       "      <td>A Mixture of Expert Based Deep Neural Network ...</td>\n",
       "      <td>ASR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>rev.com</td>\n",
       "      <td>miguel.delrio@rev.com</td>\n",
       "      <td>2104.11348</td>\n",
       "      <td>Earnings-21: A Practical Benchmark for ASR in ...</td>\n",
       "      <td>ASR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     company_domain                       email    paper_id  \\\n",
       "708  reverieinc.com  shakti.rath@reverieinc.com  2112.01023   \n",
       "709  reverieinc.com  shakti.rath@reverieinc.com  2112.01025   \n",
       "815         rev.com       miguel.delrio@rev.com  2104.11348   \n",
       "\n",
       "                                           paper_title paper_task  \n",
       "708  A higher order Minkowski loss for improved pre...        ASR  \n",
       "709  A Mixture of Expert Based Deep Neural Network ...        ASR  \n",
       "815  Earnings-21: A Practical Benchmark for ASR in ...        ASR  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_df[company_df[\"company_domain\"].str.contains(\"rev\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba7f6fbd",
   "metadata": {},
   "source": [
    "Good afternoon,\n",
    "\n",
    "sorry for the cold email, but we got your address from one of your recent arxiv papers ([paper_name or paper_id]). We are a young company, suno.ai, specializing in audio and speech data sourcing and labeling. We are fast, accurate and are familiar with audio tasks across the board (from code-switched ASR over speech enhancement to emotion recognition). If you are available, we would love to find time for a quick chat to see if Suno could be helpful to you in any way.\n",
    "\n",
    "Best,\n",
    "Suno Team\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c99ca330",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "374e91dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "34d1b094",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # check that we didn't eliminate anything bad\n",
    "# a = []\n",
    "# for domain, v in domain_to_paper_ids.items():\n",
    "#     if re.search(r\"\\.(?:edu|org|ac)\\b\", domain):\n",
    "#         a.append((domain, len(v)))\n",
    "# a = sorted(a, key=lambda x: x[-1], reverse=True)\n",
    "\n",
    "# b = []\n",
    "# for domain, v in domain_to_paper_ids.items():\n",
    "#     if not re.search(r\"\\.(?:com|ai)\\b\", domain):\n",
    "#         b.append((domain, len(v)))\n",
    "# b = sorted(b, key=lambda x: x[-1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "1c37ced9",
   "metadata": {},
   "outputs": [],
   "source": [
    "company_data = []\n",
    "for domain, task_to_paper_ids in company_map.items():\n",
    "    tmp = [domain]\n",
    "    all_papers = set()\n",
    "    for task in TASK_SEARCH_PTNS.keys():\n",
    "        if task in task_to_paper_ids:\n",
    "            paper_ids = task_to_paper_ids[task]\n",
    "            # sort papers by date\n",
    "            paper_ids = sorted(paper_ids, key=lambda x: int(x.replace(\".\", \"\")), reverse=True)\n",
    "            all_papers |= set(paper_ids)\n",
    "            tmp.append(\";\".join(paper_ids))\n",
    "        else:\n",
    "            tmp.append(None)\n",
    "    tmp.append(len(all_papers))\n",
    "    company_data.append(tmp)\n",
    "company_df = pd.DataFrame(company_data, columns=[\"domain\"] + TASK_LIST + [\"n_papers\"])\n",
    "company_df = company_df.sort_values(by=\"n_papers\", ascending=False)\n",
    "company_df = company_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "32bf5c3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"max_colwidth\", 25)\n",
    "company_df.head()\n",
    "pd.set_option(\"max_colwidth\", 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "a273f1d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>ASR</th>\n",
       "      <th>TTS</th>\n",
       "      <th>Diarization</th>\n",
       "      <th>Speech Enh.</th>\n",
       "      <th>Misc</th>\n",
       "      <th>n_papers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>comcast.com</td>\n",
       "      <td>1812.07754</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>dreamfacetech.com</td>\n",
       "      <td>2106.08468</td>\n",
       "      <td>2106.08468</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>laboro.ai</td>\n",
       "      <td>2103.14736</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>verisk.com</td>\n",
       "      <td>2110.00046</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>thoughtworks.com</td>\n",
       "      <td>2107.07402</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                domain         ASR         TTS Diarization Speech Enh.  Misc  \\\n",
       "220        comcast.com  1812.07754        None        None        None  None   \n",
       "221  dreamfacetech.com  2106.08468  2106.08468        None        None  None   \n",
       "222          laboro.ai  2103.14736        None        None        None  None   \n",
       "223         verisk.com  2110.00046        None        None        None  None   \n",
       "224   thoughtworks.com  2107.07402        None        None        None  None   \n",
       "\n",
       "     n_papers  \n",
       "220         1  \n",
       "221         1  \n",
       "222         1  \n",
       "223         1  \n",
       "224         1  "
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "35eac958",
   "metadata": {},
   "outputs": [],
   "source": [
    "company_df.to_csv(\"arxiv_company_outreach.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "c9371180",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>ASR</th>\n",
       "      <th>TTS</th>\n",
       "      <th>Diarization</th>\n",
       "      <th>Speech Enh.</th>\n",
       "      <th>Misc</th>\n",
       "      <th>n_papers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>kensho.com</td>\n",
       "      <td>2104.02014;2005.04290</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        domain                    ASR   TTS Diarization Speech Enh.  Misc  \\\n",
       "68  kensho.com  2104.02014;2005.04290  None        None        None  None   \n",
       "\n",
       "    n_papers  \n",
       "68         2  "
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_df[company_df[\"domain\"] == \"kensho.com\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "480da545",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>ASR</th>\n",
       "      <th>TTS</th>\n",
       "      <th>Diarization</th>\n",
       "      <th>Speech Enh.</th>\n",
       "      <th>Misc</th>\n",
       "      <th>n_papers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [domain, ASR, TTS, Diarization, Speech Enh., Misc, n_papers]\n",
       "Index: []"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_df[company_df[\"domain\"] == \"synthesia.io\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "bd69ac1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>ASR</th>\n",
       "      <th>TTS</th>\n",
       "      <th>Diarization</th>\n",
       "      <th>Speech Enh.</th>\n",
       "      <th>Misc</th>\n",
       "      <th>n_papers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>georgian.io</td>\n",
       "      <td>2103.15760</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         domain         ASR   TTS Diarization Speech Enh.  Misc  n_papers\n",
       "92  georgian.io  2103.15760  None        None        None  None         1"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_df[company_df[\"domain\"].str.contains(\"\\.io\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "35ebdd7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>ASR</th>\n",
       "      <th>TTS</th>\n",
       "      <th>Diarization</th>\n",
       "      <th>Speech Enh.</th>\n",
       "      <th>Misc</th>\n",
       "      <th>n_papers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [domain, ASR, TTS, Diarization, Speech Enh., Misc, n_papers]\n",
       "Index: []"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_df[company_df[\"domain\"] == \"twilio.com\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f175674a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "972d3d4b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c42746e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}