{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5c0bfda6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: we got something like 800k, but then it started stalling, should continue at some point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee339374",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "import feedparser\n",
    "import time\n",
    "import random\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import multiprocessing\n",
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import html\n",
    "\n",
    "from suno_utils.utils.text import normalize_whitespace\n",
    "\n",
    "DATA_DIR = \"/mnt/data-ssd-1/data/podcasts/bulk_rss_feeds/raw_feeds/\"\n",
    "COUNTER_DIR = \"/mnt/data-ssd-1/data/podcasts/bulk_rss_feeds/tmp__counter/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b075a57d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "065016a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2e354cef",
   "metadata": {},
   "source": [
    "## Download feeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "238cb007",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get podcast index from here: https://podcastindex.org/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "eacadcfb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0/4037236 to go\n"
     ]
    }
   ],
   "source": [
    "import sqlite3\n",
    "\n",
    "con = sqlite3.connect(\"/mnt/data-ssd-1/data/podcasts/meta/podcastindex_feeds.db\")\n",
    "cur = con.cursor()\n",
    "main_df = pd.read_sql(\"SELECT * FROM podcasts;\", con)\n",
    "con.close()\n",
    "\n",
    "# exclude existing ones\n",
    "success_podcast_ids = set([int(fp.split(\".\")[0]) for fp in os.listdir(DATA_DIR) if fp.endswith(\".feed\")])\n",
    "failed_podcast_ids = set([int(fp.split(\".\")[0]) for fp in os.listdir(COUNTER_DIR) if fp.endswith(\".txt\")])\n",
    "todo_df = main_df[~main_df[\"id\"].isin(success_podcast_ids |  failed_podcast_ids)]\n",
    "todo_df = todo_df.sample(frac=1.0, random_state=6006).reset_index(drop=True)\n",
    "work_items = list(zip(todo_df[\"id\"].astype(str), todo_df[\"url\"]))\n",
    "\n",
    "print(f\"{todo_df.shape[0]}/{main_df.shape[0]} to go\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1b8af3d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def harvest_item(work_item):\n",
    "    podcast_id, url = work_item\n",
    "    reason = \"\"\n",
    "    try:\n",
    "        r = requests.get(url, timeout=5)\n",
    "        if r.ok:\n",
    "            with open(DATA_DIR + f\"{podcast_id}.feed\", \"wb\") as f:\n",
    "                f.write(r.content)\n",
    "            return podcast_id, True\n",
    "        else:\n",
    "            reason = r.reason\n",
    "    except:\n",
    "        reason = \"Exception\"\n",
    "        pass\n",
    "    with open(COUNTER_DIR + f\"{podcast_id}.txt\", \"w\") as f:\n",
    "        f.write(reason)\n",
    "    time.sleep(0.3 + random.random() * 0.2)\n",
    "    return podcast_id, False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f743f5dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "p = multiprocessing.Pool(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c05adcb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "out = p.map(harvest_item, work_items, chunksize=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47ecdb7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "p.close()\n",
    "p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "079fa20a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls /mnt/data-ssd-1/data/podcasts/raw_rss_feeds/ | wc -l\n",
    "# 3802061"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f1676caf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls /mnt/data-ssd-1/data/podcasts/tmp__counter/ | wc -l\n",
    "# 235175"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a627ec13",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "67da50c1",
   "metadata": {},
   "source": [
    "## Process feeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "57ac4d1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tqdm\n",
    "import json\n",
    "import funcy\n",
    "import numpy as np\n",
    "import multiprocessing\n",
    "from suno_utils.utils.podcasts import load_podcast_db, load_rss_feed\n",
    "\n",
    "\n",
    "RSS_DIR = \"/mnt/data-ssd-1/data/podcasts/bulk_rss_feeds/\"\n",
    "\n",
    "RAW_RSS_DIR = os.path.join(RSS_DIR, \"raw_feeds\")\n",
    "PARSED_RSS_DIR = os.path.join(RSS_DIR, \"parsed_feeds\")\n",
    "\n",
    "def _parse_and_write(filepath):\n",
    "    try:\n",
    "        feed_info = load_rss_feed(filepath)\n",
    "    except:\n",
    "        return None\n",
    "    podcast_id = filepath.split(\"/\")[-1].split(\".\")[0]\n",
    "    out_path = os.path.join(PARSED_RSS_DIR, f\"{podcast_id}.json\")\n",
    "    with open(out_path, \"w\") as f:\n",
    "        json.dump(feed_info, f)\n",
    "\n",
    "filepaths = [os.path.join(RAW_RSS_DIR, fn) for fn in os.listdir(RAW_RSS_DIR)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d6d9e1ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 39/39 [1:39:56<00:00, 153.75s/it]\n"
     ]
    }
   ],
   "source": [
    "p = multiprocessing.Pool(20)\n",
    "_ = p.map(_parse_and_write, filepaths, chunksize=100)\n",
    "p.close()\n",
    "p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d24493f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_log_dir  parsed_feeds\traw_feeds\r\n"
     ]
    }
   ],
   "source": [
    "!ls /mnt/data-ssd-1/data/podcasts/bulk_rss_feeds/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cfc96663",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86G\t/mnt/data-ssd-1/data/podcasts/bulk_rss_feeds/parsed_feeds\r\n"
     ]
    }
   ],
   "source": [
    "!du -hs /mnt/data-ssd-1/data/podcasts/bulk_rss_feeds/parsed_feeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5743500",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c200d022",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e71aa035",
   "metadata": {},
   "source": [
    "## Search through feeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18431b2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from suno_utils.utils.podcasts import find_in_raw_feeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "d891e837",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import multiprocessing\n",
    "\n",
    "\n",
    "def foo(filepath):\n",
    "    with open(filepath) as f:\n",
    "        feed_info = json.load(f)\n",
    "    podcast_id = filepath.split(\"/\")[-1].split(\".\")[0]\n",
    "    if (str(feed_info.get(\"language\")).lower() == \"en-in\") or (\"hinglish\" in str(feed_info).lower()):\n",
    "        return podcast_id, feed_info\n",
    "    return None\n",
    "\n",
    "\n",
    "PARSED_FEEDS_DIR = \"/mnt/data-ssd-1/data/podcasts/bulk_rss_feeds/parsed_feeds/\"\n",
    "filepaths = [os.path.join(PARSED_FEEDS_DIR, filename) for filename in os.listdir(PARSED_FEEDS_DIR)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "e066bf72",
   "metadata": {},
   "outputs": [],
   "source": [
    "p = multiprocessing.Pool(20)\n",
    "out = p.map(foo, filepaths, chunksize=100)\n",
    "out = [e for e in out if e is not None]\n",
    "p.close()\n",
    "p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "4544ab76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "466"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = []\n",
    "for e in out:\n",
    "    lang = e[-1].get(\"language\")\n",
    "    if lang is None:\n",
    "        continue\n",
    "    if \"en\" in lang or \"in\" in lang:\n",
    "        data.append(e[-1])\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "131652a6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8386b581",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: exclude things like music\n",
    "\n",
    "# import re\n",
    "\n",
    "# excluded_terms = [\n",
    "#     \"music\", \"song\", \"teach\", \"poem\", \"poetr\", \"learn\", \"storie\", \"story\"\n",
    "# ]\n",
    "\n",
    "# eids = []\n",
    "# for e in out:\n",
    "#     if re.search(r\"|\".join([re.escape(e.lower()) for e in excluded_terms]), e[2], flags=re.IGNORECASE):\n",
    "#         continue\n",
    "#     if n_episode_map[e[0]] < 3:\n",
    "#         continue\n",
    "#     eids.append(e[0])\n",
    "#     print(e[0], \"-\", url_map[e[0]])\n",
    "#     print(e[1], \"-\", e[2])\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2c4a0c0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "237dc9ad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1656522c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ecb0654",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f7d727f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc99180e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a058901",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9541592",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}