{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "af42ede4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import base64\n",
    "import os\n",
    "import re\n",
    "import time\n",
    "import random\n",
    "import json\n",
    "import tqdm\n",
    "import uuid\n",
    "import requests\n",
    "import string\n",
    "import funcy\n",
    "import gzip\n",
    "import urllib\n",
    "import numpy as np\n",
    "import multiprocessing\n",
    "from bs4 import BeautifulSoup\n",
    "from contextlib import redirect_stderr, redirect_stdout, contextmanager\n",
    "import signal\n",
    "\n",
    "import pandas as pd\n",
    "import cloudscraper\n",
    "import youtube_dl\n",
    "\n",
    "from suno_utils.audio import Audio\n",
    "from suno_utils.audio.conversion import get_audio_properties, get_duration_s\n",
    "from suno_utils.utils.text import make_unique_list\n",
    "\n",
    "\n",
    "def _courtesy_sleep(avg_sleep_dur_s=0.5):\n",
    "    time.sleep((0.5 + random.random() / 2) * avg_sleep_dur_s)\n",
    "\n",
    "\n",
    "class TimeoutException(Exception): \n",
    "    pass\n",
    "\n",
    "\n",
    "@contextmanager\n",
    "def time_limit(seconds):\n",
    "    seconds = int(round(seconds))\n",
    "    def signal_handler(signum, frame):\n",
    "        raise TimeoutException(\"Function call timed out!\")\n",
    "    signal.signal(signal.SIGALRM, signal_handler)\n",
    "    signal.alarm(seconds)\n",
    "    try:\n",
    "        yield\n",
    "    finally:\n",
    "        signal.alarm(0)\n",
    "        \n",
    "    \n",
    "class Logger():\n",
    "    def __init__(self, filepath):\n",
    "        self._filepath = filepath\n",
    "        self._reset_log()\n",
    "        \n",
    "    def _reset_log(self):\n",
    "        with open(self._filepath, \"w\") as f:\n",
    "            f.write(\"\")\n",
    "        \n",
    "    def _add_line(self, line):\n",
    "        with open(self._filepath, \"a\") as f:\n",
    "            f.write(line + \"\\n\")\n",
    "\n",
    "def _parse_duration(s):\n",
    "    if s is None:\n",
    "        return 0\n",
    "    parts = s.split(\":\")\n",
    "    if len(parts) == 3:\n",
    "        nh, nm, ns = parts\n",
    "    elif len(parts) == 2:\n",
    "        nm, ns = parts\n",
    "        nh = 0\n",
    "    else:\n",
    "        raise ValueError(\"\")\n",
    "    return int(nh) * 60**2 + int(nm) * 60 + int(ns)\n",
    "    \n",
    "\n",
    "def mp_scrape(\n",
    "    extract_f, \n",
    "    queue_items, \n",
    "    global_info=None,\n",
    "    result_filepath=None, \n",
    "    log_filepath=None, \n",
    "    n_cores=5, \n",
    "    chunksize=500, \n",
    "    n_retries=3,\n",
    "    append_results=False,\n",
    "    backoff_dur_s=1.0,\n",
    "    inner_chunksize=1, \n",
    "    quiet=False,\n",
    "):\n",
    "    if global_info is not None:\n",
    "        _f = funcy.partial(extract_f, global_info=global_info)\n",
    "    else:\n",
    "        _f = extract_f\n",
    "    if result_filepath is not None and not append_results:\n",
    "        with open(result_filepath, \"w\") as f:\n",
    "            f.write(\"\")\n",
    "    if log_filepath is not None:\n",
    "        logger = Logger(log_filepath)\n",
    "    out = []\n",
    "    n_chunks = int(np.ceil(len(queue_items) / chunksize))\n",
    "    for n_chunk, queue_items_chunk in tqdm.tqdm(\n",
    "        enumerate(funcy.chunks(chunksize, queue_items)), \n",
    "        total=n_chunks,\n",
    "        disable=quiet,\n",
    "    ):\n",
    "        p = multiprocessing.Pool(n_cores)\n",
    "        t0 = time.time()\n",
    "        remaining_items = [(idx, queue_item) for idx, queue_item in enumerate(queue_items_chunk)]\n",
    "        out_chunk = [None] * len(queue_items_chunk)\n",
    "        for n_retry in range(n_retries):\n",
    "            tmp_out = p.map(_f, [queue_item for _, queue_item in remaining_items], chunksize=inner_chunksize)\n",
    "            _courtesy_sleep(avg_sleep_dur_s=backoff_dur_s)\n",
    "            tmp_remaining_items = []\n",
    "            for (idx, queue_item), tmp_out_item in zip(remaining_items, tmp_out):\n",
    "                out_chunk[idx] = tmp_out_item\n",
    "                if (\n",
    "                    isinstance(tmp_out_item, dict) and (\n",
    "                        (tmp_out_item.get(\"retry\")) == True or \n",
    "                        (\"retry\" not in tmp_out_item and tmp_out_item.get(\"success\") == False)\n",
    "                    )\n",
    "                ):\n",
    "                    tmp_remaining_items.append((idx, queue_item))\n",
    "                    continue\n",
    "            remaining_items = tmp_remaining_items\n",
    "            if len(remaining_items) == 0:\n",
    "                break\n",
    "            if n_retry < n_retries - 1:\n",
    "                logger._add_line(f\"  retrying for {len(remaining_items)}/{len(out_chunk)} items\")\n",
    "            \n",
    "        # show how many failed\n",
    "        n_failed = len([\n",
    "            e for e in out_chunk if e is None or (isinstance(e, dict) and e.get(\"success\") == False)\n",
    "        ])\n",
    "        logger._add_line(f\"  failed on {n_failed}/{len(out_chunk)} items\")\n",
    "            \n",
    "        # break if none were successful\n",
    "        if all([e is None or (isinstance(e, dict) and e.get(\"success\") == False) for e in out_chunk]):\n",
    "            logger._add_line(f\"{len(out_chunk)}/{len(out_chunk)} items failed, aborting.\")\n",
    "            \n",
    "        if result_filepath is not None:\n",
    "            with open(result_filepath, \"a\") as f:\n",
    "                for e in out_chunk:\n",
    "                    f.write(json.dumps(e) + \"\\n\")\n",
    "        else:\n",
    "            out.extend(out_chunk)\n",
    "        td = time.time() - t0\n",
    "        if log_filepath is not None:\n",
    "            logger._add_line(f\"{n_chunk+1}/{n_chunks} - last step took {round(td / 60, 1)} mins\")\n",
    "        p.close()\n",
    "        p.join()\n",
    "    logger._add_line(f\"done!\")\n",
    "    if result_filepath is not None:\n",
    "        return None\n",
    "    return out\n",
    "\n",
    "US_PROXY = (\n",
    "     \"http://brd-customer-hl_98887cab-zone-us_proxy-route_err-block-country-us:\" +\n",
    "     \"3if5he8elfe7@zproxy.lum-superproxy.io:22225\"\n",
    ")\n",
    "\n",
    "DATA_DIR = \"/data/suno/data/harvest/genius\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1037d71",
   "metadata": {},
   "source": [
    "## Collect extra meta info like views"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b72763fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lang and lang_lyris are the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7381bf94",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5480217/5480217 [04:06<00:00, 22203.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2705405 song details with youtube\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "seen_slugs = set()\n",
    "with open(os.path.join(DATA_DIR, \"song_details_with_youtube.jsonl\"), \"w\") as f:\n",
    "    f.write(\"\")\n",
    "with open(os.path.join(DATA_DIR, \"song_details.jsonl\")) as f:\n",
    "    for line in tqdm.tqdm(f, total=5480217):\n",
    "        line = line.strip()\n",
    "        if len(line) == 0:\n",
    "            continue\n",
    "        m = json.loads(line)\n",
    "        if m is None:\n",
    "            continue\n",
    "        if not m[\"success\"]:\n",
    "            continue\n",
    "        if m[\"song_slug\"] in seen_slugs:\n",
    "            continue\n",
    "        youtube_url = m[\"meta\"][\"youtube_url\"]\n",
    "        # almost all agree, so let's skip the ones that don't\n",
    "        if youtube_url is None or youtube_url != m[\"meta\"][\"youtube_url_2\"]:\n",
    "            continue\n",
    "        youtube_id = youtube_url.split(\"=\")[-1]\n",
    "        if len(youtube_id) != 11:\n",
    "            continue\n",
    "        try:\n",
    "            youtube_start = float(m[\"meta\"][\"youtube_start\"])\n",
    "        except:\n",
    "            continue\n",
    "        new_m = {\n",
    "            \"genius_slug\": m[\"song_slug\"],\n",
    "            \"youtube_id\": youtube_id,\n",
    "            \"youtube_start\": youtube_start,\n",
    "            \"lang\": m[\"meta\"][\"lang\"],\n",
    "            \"views\": m[\"meta\"][\"views\"],\n",
    "            \"tags\": m[\"meta\"][\"tags\"],\n",
    "            \"lyrics\": m[\"meta\"][\"lyrics\"],\n",
    "        }\n",
    "        with open(os.path.join(DATA_DIR, \"song_details_with_youtube.jsonl\"), \"a\") as f:\n",
    "            f.write(json.dumps(new_m) + \"\\n\")\n",
    "        seen_slugs.add(m[\"song_slug\"])\n",
    "print(len(seen_slugs), \"song details with youtube\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abf21eb5",
   "metadata": {},
   "source": [
    "## Download youtube audio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "72fd1cb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2705405 items loaded.\n"
     ]
    }
   ],
   "source": [
    "youtube_ids = []\n",
    "with open(os.path.join(DATA_DIR, \"song_details_with_youtube.jsonl\")) as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if len(line) == 0:\n",
    "            continue\n",
    "        youtube_ids.append(json.loads(line)[\"youtube_id\"])\n",
    "print(len(youtube_ids), \"items loaded.\")\n",
    "# 2705405 items loaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea3a6380",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # (optional) remove all ids that were processed (unless temporary fail)\n",
    "# handled_ids = set()\n",
    "# with open(os.path.join(DATA_DIR, \"youtube_metas.jsonl\")) as f:\n",
    "#     for line in f:\n",
    "#         line = line.strip()\n",
    "#         if len(line) == 0:\n",
    "#             continue\n",
    "#         m = json.loads(line)\n",
    "#         if (\n",
    "#             m[\"success\"] or (\n",
    "#                 \"unable to download video data\" not in m[\"fail_message\"] and\n",
    "#                 \"No video formats found\" not in m[\"fail_message\"]\n",
    "#             )\n",
    "#         ):\n",
    "#             handled_ids.add(m[\"id\"])\n",
    "# print(len(handled_ids), \"handled items.\")\n",
    "# youtube_ids = [youtube_id for youtube_id in youtube_ids if youtube_id not in handled_ids]\n",
    "# print(len(youtube_ids), \"remaining items.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3138eecd",
   "metadata": {},
   "outputs": [],
   "source": [
    "YDL_OPTS = {\n",
    "    \"format\": \"worstaudio\",\n",
    "    \"outtmpl\": os.path.join(DATA_DIR, \"audio\", \"%(id)s.%(ext)s\"),\n",
    "    \"writesubtitles\": True,\n",
    "    \"allsubtitles\": True,\n",
    "    \"subtitlesformat\": \"best\",\n",
    "    \"socket_timeout\": 5.0,\n",
    "    \"proxy\": US_PROXY,\n",
    "}\n",
    "MAX_DOWNLOAD_TIME_S = 60\n",
    "\n",
    "YOUTUBE_BASE_URL = \"https://www.youtube.com/watch?v=\"\n",
    "\n",
    "def _resolve_youtube(youtube_id):\n",
    "    url = YOUTUBE_BASE_URL + youtube_id\n",
    "    out_data = {\n",
    "        \"id\": youtube_id,\n",
    "    }\n",
    "    t0 = time.time()\n",
    "    try:\n",
    "        with redirect_stderr(open(os.devnull, \"w\")):\n",
    "            with redirect_stdout(open(os.devnull, \"w\")):\n",
    "                with youtube_dl.YoutubeDL(YDL_OPTS) as ydl:\n",
    "                    # pre-download to check if we want it (duration, views etc)\n",
    "#                     info = ydl.extract_info(url, download=False)\n",
    "                    if MAX_DOWNLOAD_TIME_S is not None:\n",
    "                        with time_limit(MAX_DOWNLOAD_TIME_S):\n",
    "                            info = ydl.extract_info(url, download=True)\n",
    "                    else:\n",
    "                        info = ydl.extract_info(url, download=True)\n",
    "        _ = info.pop(\"formats\", None)\n",
    "        _ = info.pop(\"thumbnails\", None)\n",
    "        out_data[\"success\"] = True\n",
    "        out_data[\"retry\"] = False\n",
    "        out_data[\"meta\"] = info\n",
    "        out_data[\"audio_filename\"] = f\"{info['id']}.{info['ext']}\"\n",
    "    except Exception as e:\n",
    "        out_data[\"success\"] = False\n",
    "        out_data[\"retry\"] = (\n",
    "            \"TimeoutException\" in str(type(e)) or\n",
    "            \"unable to download video data\" in str(e) or \n",
    "            \"No video formats found\" in str(e)\n",
    "        )\n",
    "        out_data[\"fail_type\"] = str(type(e))\n",
    "        out_data[\"fail_message\"] = str(e)\n",
    "    t1 = time.time()\n",
    "    out_data[\"runtime_s\"] = round(t1 - t0, 1)\n",
    "    return out_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1be659db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/data/suno/data/harvest/genius/youtube_metas.jsonl'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.join(DATA_DIR, \"youtube_metas.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "edb937ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wc -l /data/suno/data/harvest/genius/youtube_metas.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "edac8d29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: correct fail reporting\n",
    "# TODO: stop if all fail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "87d9217d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 413/413 [67:26:20<00:00, 587.85s/it]\n"
     ]
    }
   ],
   "source": [
    "# ~9mins for 5k videos, ~4 days for 2.7million (~100k hours) at 100 cores - ~4Tb\n",
    "_ = mp_scrape(\n",
    "    _resolve_youtube, \n",
    "    youtube_ids[645025:], \n",
    "    chunksize=5000,\n",
    "    n_cores=100,\n",
    "    n_retries=3,\n",
    "    append_results=True,\n",
    "    result_filepath=os.path.join(DATA_DIR, \"youtube_metas.jsonl\"),\n",
    "    log_filepath=os.path.join(DATA_DIR, \"logs\", \"audio_harvest.log\"),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f443c2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: first 50 should be fast-ish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "af2ce70e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls -1 /data/suno/data/harvest/genius/audio | grep -v vtt | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59ea1a2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ffprobe -v error -select_streams a:0 -show_entries stream=codec_name -of default=nokey=1:noprint_wrappers=1 test.webm\n",
    "# !ffmpeg -i test.webm -vn -map 0:a -acodec copy test.opus  # 0:a smooshes, 0:a:0 takes first audio\n",
    "# looks like all streams have just one audio track. wemb is opus and m4a is aac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99736dd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# first 92 steps should go fast, with slightly lower fail rate than ~20%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa5461a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c0ffc215",
   "metadata": {},
   "source": [
    "## compile full info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "8ec306cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2609003 song details\n"
     ]
    }
   ],
   "source": [
    "# get genius info map\n",
    "genius_meta = {}\n",
    "with open(os.path.join(DATA_DIR, \"song_details_with_youtube.jsonl\")) as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if len(line) == 0:\n",
    "            continue\n",
    "        m = json.loads(line)\n",
    "        if m is None or m[\"youtube_id\"] is None:\n",
    "            continue\n",
    "        genius_meta[m[\"youtube_id\"]] = {\n",
    "            \"lyrics\": m[\"lyrics\"],\n",
    "            \"meta\": {\n",
    "                \"slug\": m[\"genius_slug\"],\n",
    "                \"youtube_start\": m[\"youtube_start\"],\n",
    "                \"lang\": m[\"lang\"],\n",
    "                \"views\": m[\"views\"],\n",
    "                \"tags\": m[\"tags\"],\n",
    "            },\n",
    "        }\n",
    "print(len(genius_meta), \"song details\")\n",
    "# 2609003 song details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "18e0acbe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79905 audios with subtitles\n"
     ]
    }
   ],
   "source": [
    "# get subtitle map\n",
    "from collections import defaultdict\n",
    "subtitle_map = defaultdict(list)\n",
    "for fn in os.listdir(os.path.join(DATA_DIR, \"audio\")):\n",
    "    if fn.split(\".\")[-1] == \"vtt\":\n",
    "        youtube_id, _, _ = fn.split(\".\")\n",
    "        subtitle_map[youtube_id].append(os.path.join(DATA_DIR, \"audio\", fn))\n",
    "print(len(subtitle_map), \"audios with subtitles\")\n",
    "# 79905 audios with subtitles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "7400bd5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2705405it [31:09, 1447.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2295501 pairs found\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "n_pairs = 0\n",
    "durations_s = []\n",
    "with open(os.path.join(DATA_DIR, \"pairs_metadata.jsonl\"), \"w\") as fw:\n",
    "    with open(os.path.join(DATA_DIR, \"youtube_metas.jsonl\")) as f:\n",
    "        for line in tqdm.tqdm(f, total=2_295_501):\n",
    "            line = line.strip()\n",
    "            if len(line) == 0:\n",
    "                continue\n",
    "            m = json.loads(line)\n",
    "            if not m[\"success\"]:\n",
    "                continue\n",
    "            if not os.path.exists(os.path.join(DATA_DIR, \"audio\", m[\"audio_filename\"])):\n",
    "                continue\n",
    "            if m[\"meta\"][\"id\"] not in genius_meta:\n",
    "                continue\n",
    "            meta_info = {\n",
    "                \"audio_filepath\": os.path.join(DATA_DIR, \"audio\", m[\"audio_filename\"]),\n",
    "                \"lyrics\": genius_meta[m[\"meta\"][\"id\"]][\"lyrics\"],\n",
    "                \"subtitles_filepaths\": subtitle_map.get(m[\"meta\"][\"id\"], []),\n",
    "                \"genius_meta\": genius_meta[m[\"meta\"][\"id\"]][\"meta\"],\n",
    "                \"youtube_meta\": {\n",
    "                    \"id\": m[\"meta\"][\"id\"],\n",
    "                    \"title\": m[\"meta\"][\"title\"],\n",
    "                    \"description\": m[\"meta\"][\"description\"],\n",
    "                    \"duration\": m[\"meta\"][\"duration\"],\n",
    "                    \"view_count\": m[\"meta\"][\"view_count\"],\n",
    "                    \"like_count\": m[\"meta\"].get(\"like_count\"),\n",
    "                    \"upload_date\": m[\"meta\"][\"upload_date\"],\n",
    "                    \"uploader_id\": m[\"meta\"][\"uploader_id\"],\n",
    "                    \"channel_id\": m[\"meta\"][\"channel_id\"],\n",
    "                    \"categories\": m[\"meta\"][\"categories\"],\n",
    "                    \"tags\": m[\"meta\"][\"tags\"],\n",
    "                    \"is_live\": m[\"meta\"][\"is_live\"],\n",
    "                    \"format\": {\n",
    "                        \"filesize\": m[\"meta\"][\"filesize\"],\n",
    "                        \"asr\": m[\"meta\"][\"asr\"],\n",
    "                        \"format_id\": m[\"meta\"][\"format_id\"],\n",
    "                        \"acodec\": m[\"meta\"][\"acodec\"],\n",
    "                        \"abr\": m[\"meta\"].get(\"abr\"),\n",
    "                        \"container\": m[\"meta\"][\"container\"],\n",
    "                    },\n",
    "                },\n",
    "            }\n",
    "            fw.write(json.dumps(meta_info) + \"\\n\")\n",
    "            n_pairs += 1\n",
    "            durations_s.append(m[\"meta\"][\"duration\"])\n",
    "print(n_pairs, \"pairs found\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8dcca82",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Upload to s3\n",
    "# s4cmd dsync -r /data/suno/data/harvest/genius s3://suno-data/datasets/harvest/genius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9b43dd65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "140832.6 hours of data in total\n",
      "129049.8 hours of reliable data\n"
     ]
    }
   ],
   "source": [
    "print(round(np.sum(durations_s) / 60 / 60, 1), \"hours of data in total\")\n",
    "print(round(np.sum([d for d in durations_s if d <= 8 * 60 and d >= 1 * 60]) / 60 / 60, 1), \"hours of reliable data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10e077eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 129,050 hours of genius data\n",
    "# en              62.6%\n",
    "# es               5.7%\n",
    "# pt               4.2%\n",
    "# fr               4.1%\n",
    "# ru               3.8%\n",
    "# de               3.4%\n",
    "# pl               2.7%\n",
    "# it               2.5%\n",
    "# tr               1.3%\n",
    "# ko               1.1%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baa9001a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d328c9c2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a6ca8c0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}