{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ddbc61e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fmlist.org - best worldwide (no metadata)\n",
    "# tunein - decent worldwide coverage incl emails but not API\n",
    "# wiki - good worldwide exhaustive but no emails (details page language of country to get website)\n",
    "# publicfiles.fcc.gov - good US incl emails but no worldwide"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eee3fb68",
   "metadata": {},
   "source": [
    "## FCC data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4654993",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import urllib.parse\n",
    "import time\n",
    "import tqdm\n",
    "import random\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8bb3db5",
   "metadata": {},
   "outputs": [],
   "source": [
    "## bulk files are a bit of a mess\n",
    "# party_df = pd.read_csv(\"all_cdbs_files/party.dat\", sep=\"|\", header=None)\n",
    "# party_df = party_df[~party_df[7].isnull()].reset_index(drop=True)\n",
    "# df = pd.read_csv(\"all_cdbs_files/facility.dat\", sep=\"|\",  header=None)\n",
    "# df = df[df[10].isin([\"FM\", \"AM\"])].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9dc2c8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "HEADERS = {\n",
    "    \"User-Agent\": (\n",
    "        \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) \"\n",
    "        \"Chrome/50.0.2661.102 Safari/537.36\"\n",
    "    )\n",
    "}\n",
    "\n",
    "sess = requests.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fff755b",
   "metadata": {},
   "outputs": [],
   "source": [
    "overview_utl_ptn = \"https://publicfiles.fcc.gov/api/service/{}/facility/getall.json\"\n",
    "data = {}\n",
    "for station_type in [\"fm\", \"am\", \"tv\"]:\n",
    "    r_json = sess.get(url=overview_utl_ptn.format(station_type), headers=HEADERS).json()\n",
    "    data[station_type] = r_json\n",
    "    print(station_type, \"-\", r_json[\"message\"])\n",
    "    time.sleep(random.random() * 0.5)\n",
    "# fm - 11261 Facilities Returned\n",
    "# am - 4690 Facilities Returned\n",
    "# tv - 3688 Facilities Returned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "7ff9d950",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting fm data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11261/11261 [1:10:31<00:00,  2.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting am data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4690/4690 [31:28<00:00,  2.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting tv data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3688/3688 [22:17<00:00,  2.76it/s]\n"
     ]
    }
   ],
   "source": [
    "detail_utl_ptn = \"https://publicfiles.fcc.gov/api/service/{}/facility/id/{}.json\"\n",
    "\n",
    "# detail_data = {}\n",
    "n = 0\n",
    "for station_type in [\"fm\", \"am\", \"tv\"]:\n",
    "    print(\"getting {} data\".format(station_type))\n",
    "    if station_type not in detail_data:\n",
    "        detail_data[station_type] = {}\n",
    "    for facility in tqdm.tqdm(data[station_type][\"results\"]['facilityList']):\n",
    "        if facility[\"id\"] in detail_data[station_type]:\n",
    "            continue\n",
    "        r = sess.get(url=detail_utl_ptn.format(station_type, facility[\"id\"]), headers=HEADERS)\n",
    "        try:\n",
    "            r_json = r.json()\n",
    "        except:\n",
    "            continue\n",
    "        detail_data[station_type][facility[\"id\"]] = r_json\n",
    "        n += 1\n",
    "        if n % 500 == 0:\n",
    "            with open(\"tmp_data.json\", \"w\") as f:\n",
    "                json.dump(detail_data, f)\n",
    "        time.sleep(random.random() * 0.2)\n",
    "with open(\"tmp_data.json\", \"w\") as f:\n",
    "    json.dump(detail_data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be47c400",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8ff62bc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8397c50",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23710144",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}