{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34fb42e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://query.wikidata.org/\n",
    "\n",
    "# SELECT ?entity_id ?enitity_name ?enitity_name_alias ?id\n",
    "# WHERE\n",
    "# {\n",
    "#   ?entity_id wdt:P673 ?id. # P604, P672\n",
    "#   ?entity_id rdfs:label ?enitity_name filter (lang(?enitity_name) = \"en\").\n",
    "#   ?entity_id skos:altLabel ?enitity_name_alias filter (lang(?enitity_name_alias) = \"en\")\n",
    "# }\n",
    "\n",
    "# P673, P604, P672 for emed, medline, mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f71540c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we want ~300k videos for 50k hours\n",
    "# search gives max 500 results per term, 20 per page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0c20e767",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7aac5f50",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"emed_terms.json\") as f:\n",
    "    emed_data = json.load(f)\n",
    "with open(\"medline_terms.json\") as f:\n",
    "    medline_data = json.load(f)\n",
    "allowed_mesh_categories = [\"A\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H02\"]\n",
    "with open(\"mesh_terms.json\") as f:\n",
    "    mesh_data = json.load(f)\n",
    "filtered_mesh_data = [e for e in mesh_data if any(e[\"id\"].startswith(ee) for ee in allowed_mesh_categories)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "bc8810b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11764 emed terms\n",
      "9403 medline terms\n",
      "64087 mesh terms\n",
      "\n",
      "66453 total terms\n"
     ]
    }
   ],
   "source": [
    "emed_terms = set(\n",
    "    [e[\"enitity_name\"].lower() for e in emed_data if len(e[\"enitity_name\"]) >= 5] + \n",
    "    [e[\"enitity_name_alias\"].lower() for e in emed_data if len(e[\"enitity_name_alias\"]) >= 5]\n",
    ")\n",
    "medline_terms = set(\n",
    "    [e[\"enitity_name\"].lower() for e in medline_data if len(e[\"enitity_name\"]) >= 5] + \n",
    "    [e[\"enitity_name_alias\"].lower() for e in medline_data if len(e[\"enitity_name_alias\"]) >= 5]\n",
    ")\n",
    "mesh_terms = set(\n",
    "    [e[\"enitity_name\"].lower() for e in filtered_mesh_data if len(e[\"enitity_name\"]) >= 5] + \n",
    "    [e[\"enitity_name_alias\"].lower() for e in filtered_mesh_data if len(e[\"enitity_name_alias\"]) >= 5]\n",
    ")\n",
    "total_terms = emed_terms | medline_terms | mesh_terms\n",
    "total_terms = list(set([\n",
    "    term.replace(\"(disorder)\", \"\").strip().replace(\"(disease)\", \"\").strip().replace(\"(finding)\", \"\").strip() \n",
    "    for term in total_terms\n",
    "]))\n",
    "\n",
    "print(len(emed_terms), \"emed terms\")\n",
    "print(len(medline_terms), \"medline terms\")\n",
    "print(len(mesh_terms), \"mesh terms\")\n",
    "print()\n",
    "print(len(total_terms), \"total terms\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "3e3a82bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"seed_terms.json\", \"w\") as f:\n",
    "    json.dump(total_terms, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f9e89a6c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['azithromycin anhydrous', 'azithromycin', 'azithromycine', 'azithromycinum']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[e for e in total_terms if \"azithromycin\" in e]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "9a6cb3e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['brain pacemaker',\n",
       " 'pacemaker malfunction',\n",
       " 'cardiac pacemaker',\n",
       " 'artificial cardiac pacemaker',\n",
       " 'pacemaker',\n",
       " 'artificial pacemaker',\n",
       " 'pacemaker failure',\n",
       " 'pacemaker, artificial']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[e for e in total_terms if \"pacemaker\" in e]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef1ea408",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "539ed3aa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adb5da4c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ae83c63",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}