{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "09069055", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 105, "id": "e696d5ee", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "%matplotlib inline\n", "\n", "import os\n", "import json\n", "import requests\n", "import urllib3\n", "\n", "from bs4 import BeautifulSoup, SoupStrainer\n", "from pytube import YouTube\n", "from pytube.exceptions import VideoUnavailable\n", "from vimeo_downloader import Vimeo, RequestError\n", "import time\n", "\n", "import datetime\n", "import tqdm\n", "import glob\n" ] }, { "cell_type": "code", "execution_count": 79, "id": "8a2f6481", "metadata": {}, "outputs": [], "source": [ "HEADERS = {\n", " \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36\"\n", "}" ] }, { "cell_type": "code", "execution_count": 106, "id": "cab3d85d", "metadata": {}, "outputs": [], "source": [ "def download_youtube(url, ouput_dir, force=False):\n", " video_key = url.replace(\"https://www.youtube.com/watch?v=\", \"\")\n", " fname = video_key + \".mp4\"\n", " yt = YouTube(url)\n", " full_fpath = os.path.join(output_dir, fname)\n", " meta = {\"meta\": yt.metadata.metadata, \"description\": yt.description}\n", " try:\n", " file_type = \"audio\"\n", " s = yt.streams.filter(type=\"audio\").first()\n", " except VideoUnavailable:\n", " file_type = \"video\"\n", " print(f\"cant't find audio for {video_key}\")\n", " try:\n", " s = yt.streams.first()\n", " except:\n", " s = None\n", " file_type = None\n", " metadata[video_key] = yt.description\n", " if s is not None:\n", " if os.path.exists(full_fpath) and not force:\n", " print(f\"found file {full_fpath}, skipping...\")\n", " else:\n", " print(f\"downloading {link} to {full_fpath}\")\n", " s.download(download_directory=output_dir, filename=fname)\n", " return {\n", " \"filename\": full_fpath,\n", " \"file_type\": file_type,\n", " \"metadata\": meta,\n", " \"url\": url,\n", " }\n", "\n", "\n", "def download_vimeo(url, output_dir, force=False):\n", " v = Vimeo(url)\n", " video_key = url.replace(\"https://vimeo.com/\", \"\")\n", " fname = video_key + \".mp4\"\n", " full_fpath = os.path.join(output_dir, fname)\n", " meta = {f: getattr(v.metadata, f) for f in v.metadata._fields}\n", " s = v.streams\n", " if s:\n", " s = s[0]\n", " if isinstance(s, list):\n", " s = s[0]\n", " \n", " if os.path.exists(full_fpath) and not force:\n", " print(f\"found file {full_fpath}, skipping...\")\n", " else:\n", " print(f\"downloading {url} to {full_fpath}\")\n", " try:\n", " s.download(download_directory=output_dir, filename=fname, mute=True)\n", " except RequestError as e:\n", " print(f\"failed to download {url}\")\n", " raise e\n", " return {\"filename\": full_fpath, \"file_type\": \"\", \"metadata\": meta, \"url\": url}" ] }, { "cell_type": "code", "execution_count": null, "id": "c3d2f4cf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 107, "id": "1b8cf4f0", "metadata": {}, "outputs": [], "source": [ "def get_video_links(vid_resp):\n", " vimeo_links = []\n", " youtube_links = []\n", " if not vid_resp.ok:\n", " return \n", " for link in BeautifulSoup(vid_resp.text, parse_only=SoupStrainer('a')):\n", " if link.has_attr('href'):\n", " href = link[\"href\"]\n", " if \"youtube\" in href:\n", " youtube_links.append(href)\n", " elif \"vimeo\" in href:\n", " vimeo_links.append(href)\n", " return {\"youtube\": youtube_links, \"vimeo\": vimeo_links}\n", "\n", "def get_more_video_links(vid_resp):\n", " links = []\n", " for link in BeautifulSoup(vid_resp.text, parse_only=SoupStrainer('a')):\n", " if link.has_attr('href'):\n", " href = link[\"href\"]\n", " if href.startswith(\"?page=\"):\n", " links.append(href)\n", " return links" ] }, { "cell_type": "code", "execution_count": null, "id": "0a899743", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "822281c2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 108, "id": "2cdb118d", "metadata": {}, "outputs": [], "source": [ "BASE_URL = \"https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_\"\n", "\n", "OUTPUT_DIR = \"/mnt/data-ssd-1/data/nz-parl\"\n", "\n", "with open(os.path.join(OUTPUT_DIR, \"manifest.json\"), \"r\") as fi:\n", " manifest = json.load(fi)" ] }, { "cell_type": "code", "execution_count": 109, "id": "6b3c5c25", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "257" ] }, "execution_count": 109, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(manifest)" ] }, { "cell_type": "code", "execution_count": null, "id": "efce1035", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 116, "id": "e698d6b7", "metadata": {}, "outputs": [], "source": [ " \n", "def process_manifest_entry(res, force_redownload=False):\n", " sub_dir = (\n", " res[\"url\"]\n", " .replace(\n", " \"https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_\", \"\"\n", " )\n", " .split(\"_\")[0]\n", " )\n", " assert len(sub_dir) == 8, \"shit\"\n", " output_dir = os.path.join(OUTPUT_DIR, sub_dir)\n", " assert os.path.exists(output_dir)\n", " \n", " mp4_files = glob.glob(os.path.join(output_dir, \"*.mp4\"))\n", "# if not force_redownload:\n", "# if mp4_files:\n", "# return None, output_dir\n", " \n", "\n", " youtube_links = res[\"links\"][\"youtube\"]\n", " vimeo_links = res[\"links\"][\"vimeo\"]\n", " youtube_links = list(set(youtube_links))\n", " vimeo_links = list(set(vimeo_links))\n", "\n", " print(\n", " f\"found {len(youtube_links)} youtube and {len(vimeo_links)} vimeo links for {res['url']}\"\n", " )\n", " \n", " res_file = os.path.join(output_dir, \"results.json\")\n", " res_file2 = os.path.join(output_dir, \"results2.json\")\n", " have_all_vids = len(vimeo_links) == len(mp4_files)\n", " num_vids = len(mp4_files)\n", " if have_all_vids and (os.path.exists(res_file) or os.path.exists(res_file2)):\n", " print(f\"date {sub_dir} has all {num_vids} and a results file. Skipping\")\n", " return None, os.path.join(output_dir, \"results2.json\")\n", "\n", " results = []\n", " for vl in tqdm.tqdm_notebook(vimeo_links):\n", " # {\"filename\": full_fpath, \"file_type\": \"\", \"metadata\": meta, \"url\": url}\n", " tmp = download_vimeo(vl, output_dir)\n", " time.sleep(1)\n", " results.append(tmp)\n", " \n", " # FIXME NO YOUTUBE\n", " return results, os.path.join(output_dir, \"results2.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "c2fe3dee", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 117, "id": "1c44d8a4", "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "found 0 youtube and 54 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220317_20220317\n", "date 20220317 has all 54 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220317/results2.json\n", "found 0 youtube and 76 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220316_20220316\n", "date 20220316 has all 76 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220316/results2.json\n", "found 0 youtube and 98 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220315_20220315\n", "date 20220315 has all 98 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220315/results2.json\n", "found 0 youtube and 35 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220310_20220310\n", "date 20220310 has all 35 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220310/results2.json\n", "found 0 youtube and 74 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220309_20220309\n", "date 20220309 has all 74 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220309/results2.json\n", "found 0 youtube and 96 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220308_20220308\n", "date 20220308 has all 96 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220308/results2.json\n", "found 0 youtube and 9 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220303_20220303\n", "date 20220303 has all 9 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220303/results2.json\n", "found 0 youtube and 70 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220302_20220302\n", "date 20220302 has all 70 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220302/results2.json\n", "found 0 youtube and 70 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220301_20220301\n", "date 20220301 has all 70 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220301/results2.json\n", "found 0 youtube and 36 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220217_20220217\n", "date 20220217 has all 36 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220217/results2.json\n", "found 0 youtube and 131 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220216_20220216\n", "date 20220216 has all 131 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220216/results2.json\n", "found 0 youtube and 80 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220215_20220215\n", "date 20220215 has all 80 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220215/results2.json\n", "found 0 youtube and 80 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220215_20220215\n", "date 20220215 has all 80 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220215/results2.json\n", "found 0 youtube and 35 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220210_20220210\n", "date 20220210 has all 35 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220210/results2.json\n", "found 0 youtube and 73 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220209_20220209\n", "date 20220209 has all 73 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220209/results2.json\n", "found 0 youtube and 49 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20220208_20220208\n", "date 20220208 has all 49 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20220208/results2.json\n", "found 0 youtube and 74 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211215_20211215\n", "date 20211215 has all 74 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211215/results2.json\n", "found 0 youtube and 91 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211214_20211214\n", "date 20211214 has all 91 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211214/results2.json\n", "found 0 youtube and 91 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211214_20211214\n", "date 20211214 has all 91 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211214/results2.json\n", "found 0 youtube and 100 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211209_20211209\n", "date 20211209 has all 100 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211209/results2.json\n", "found 0 youtube and 88 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211208_20211208\n", "date 20211208 has all 88 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211208/results2.json\n", "found 0 youtube and 88 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211208_20211208\n", "date 20211208 has all 88 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211208/results2.json\n", "found 0 youtube and 77 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211207_20211207\n", "date 20211207 has all 77 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211207/results2.json\n", "found 0 youtube and 45 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211125_20211125\n", "date 20211125 has all 45 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211125/results2.json\n", "found 0 youtube and 83 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211123_20211123\n", "date 20211123 has all 83 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211123/results2.json\n", "found 0 youtube and 83 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211123_20211123\n", "date 20211123 has all 83 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211123/results2.json\n", "found 0 youtube and 87 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211118_20211118\n", "date 20211118 has all 87 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211118/results2.json\n", "found 0 youtube and 174 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211117_20211117\n", "date 20211117 has all 174 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211117/results2.json\n", "found 0 youtube and 174 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211117_20211117\n", "date 20211117 has all 174 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211117/results2.json\n", "found 0 youtube and 93 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211116_20211116\n", "date 20211116 has all 93 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211116/results2.json\n", "found 0 youtube and 32 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211111_20211111\n", "date 20211111 has all 32 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211111/results2.json\n", "found 0 youtube and 122 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211110_20211110\n", "date 20211110 has all 122 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211110/results2.json\n", "found 0 youtube and 93 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211109_20211109\n", "date 20211109 has all 93 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211109/results2.json\n", "found 0 youtube and 93 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211109_20211109\n", "date 20211109 has all 93 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211109/results2.json\n", "found 0 youtube and 99 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211028_20211028\n", "date 20211028 has all 99 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211028/results2.json\n", "found 0 youtube and 79 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211027_20211027\n", "date 20211027 has all 79 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211027/results2.json\n", "found 0 youtube and 79 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211027_20211027\n", "date 20211027 has all 79 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211027/results2.json\n", "found 0 youtube and 148 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211026_20211026\n", "date 20211026 has all 148 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211026/results2.json\n", "found 0 youtube and 41 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211021_20211021\n", "date 20211021 has all 41 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211021/results2.json\n", "found 0 youtube and 123 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211020_20211020\n", "date 20211020 has all 123 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211020/results2.json\n", "found 0 youtube and 128 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211019_20211019\n", "date 20211019 has all 128 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211019/results2.json\n", "found 0 youtube and 128 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20211019_20211019\n", "date 20211019 has all 128 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20211019/results2.json\n", "found 0 youtube and 82 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210930_20210930\n", "date 20210930 has all 82 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210930/results2.json\n", "found 0 youtube and 78 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210929_20210929\n", "date 20210929 has all 78 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210929/results2.json\n", "found 0 youtube and 78 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210929_20210929\n", "date 20210929 has all 78 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210929/results2.json\n", "found 0 youtube and 118 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210928_20210928\n", "date 20210928 has all 118 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210928/results2.json\n", "found 0 youtube and 79 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210923_20210923\n", "date 20210923 has all 79 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210923/results2.json\n", "found 0 youtube and 80 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210922_20210922\n", "date 20210922 has all 80 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210922/results2.json\n", "found 0 youtube and 77 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210921_20210921\n", "date 20210921 has all 77 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210921/results2.json\n", "found 0 youtube and 24 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210909_20210909\n", "date 20210909 has all 24 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210909/results2.json\n", "found 0 youtube and 76 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210908_20210908\n", "date 20210908 has all 76 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210908/results2.json\n", "found 0 youtube and 94 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210907_20210907\n", "date 20210907 has all 94 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210907/results2.json\n", "found 0 youtube and 10 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210902_20210902\n", "date 20210902 has all 10 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210902/results2.json\n", "found 0 youtube and 24 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210901_20210901\n", "date 20210901 has all 24 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210901/results2.json\n", "found 0 youtube and 89 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210831_20210831\n", "date 20210831 has all 89 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210831/results2.json\n", "found 0 youtube and 40 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210812_20210812\n", "date 20210812 has all 40 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210812/results2.json\n", "found 0 youtube and 97 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210811_20210811\n", "date 20210811 has all 97 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210811/results2.json\n", "found 0 youtube and 125 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210810_20210810\n", "date 20210810 has all 125 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210810/results2.json\n", "found 0 youtube and 35 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210805_20210805\n", "date 20210805 has all 35 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210805/results2.json\n", "found 0 youtube and 97 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210804_20210804\n", "date 20210804 has all 97 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210804/results2.json\n", "found 0 youtube and 187 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210803_20210803\n", "date 20210803 has all 187 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210803/results2.json\n", "found 0 youtube and 67 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210708_20210708\n", "date 20210708 has all 67 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210708/results2.json\n", "found 0 youtube and 110 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210707_20210707\n", "date 20210707 has all 110 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210707/results2.json\n", "found 0 youtube and 94 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210706_20210706\n", "date 20210706 has all 94 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210706/results2.json\n", "found 0 youtube and 42 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210701_20210701\n", "date 20210701 has all 42 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210701/results2.json\n", "found 0 youtube and 84 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210630_20210630\n", "date 20210630 has all 84 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210630/results2.json\n", "found 0 youtube and 114 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210629_20210629\n", "date 20210629 has all 114 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210629/results2.json\n", "found 0 youtube and 36 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210624_20210624\n", "date 20210624 has all 36 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210624/results2.json\n", "found 0 youtube and 112 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210623_20210623\n", "date 20210623 has all 112 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210623/results2.json\n", "found 0 youtube and 84 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210622_20210622\n", "date 20210622 has all 84 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210622/results2.json\n", "found 0 youtube and 30 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210610_20210610\n", "date 20210610 has all 30 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210610/results2.json\n", "found 0 youtube and 97 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210609_20210609\n", "date 20210609 has all 97 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210609/results2.json\n", "found 0 youtube and 62 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210608_20210608\n", "date 20210608 has all 62 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210608/results2.json\n", "found 0 youtube and 39 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210603_20210603\n", "date 20210603 has all 39 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210603/results2.json\n", "found 0 youtube and 70 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210602_20210602\n", "date 20210602 has all 70 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210602/results2.json\n", "found 0 youtube and 80 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210601_20210601\n", "date 20210601 has all 80 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210601/results2.json\n", "found 0 youtube and 108 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210520_20210520\n", "date 20210520 has all 108 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210520/results2.json\n", "found 0 youtube and 91 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210519_20210519\n", "date 20210519 has all 91 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210519/results2.json\n", "found 0 youtube and 94 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210518_20210518\n", "date 20210518 has all 94 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210518/results2.json\n", "found 0 youtube and 37 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210513_20210513\n", "date 20210513 has all 37 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210513/results2.json\n", "found 0 youtube and 75 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210512_20210512\n", "date 20210512 has all 75 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210512/results2.json\n", "found 0 youtube and 155 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210511_20210511\n", "date 20210511 has all 155 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210511/results2.json\n", "found 0 youtube and 42 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210506_20210506\n", "date 20210506 has all 42 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210506/results2.json\n", "found 0 youtube and 89 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210505_20210505\n", "date 20210505 has all 89 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210505/results2.json\n", "found 0 youtube and 143 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210504_20210504\n", "date 20210504 has all 143 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210504/results2.json\n", "found 0 youtube and 44 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210415_20210415\n", "date 20210415 has all 44 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210415/results2.json\n", "found 0 youtube and 60 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210414_20210414\n", "date 20210414 has all 60 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210414/results2.json\n", "found 0 youtube and 11 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210413_20210413\n", "date 20210413 has all 11 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210413/results2.json\n", "found 0 youtube and 38 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210408_20210408\n", "date 20210408 has all 38 and a results file. Skipping\n", "skipping download because found mp4 files in /mnt/data-ssd-1/data/nz-parl/20210408/results2.json\n", "found 0 youtube and 74 vimeo links for https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20210407_20210407\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_2624128/989311261.py:37: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n", "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n", " for vl in tqdm.tqdm_notebook(vimeo_links):\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "73734e198dc04ce1aee8d060442ac8ee", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/74 [00:00\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"results.json\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/mnt/data-ssd-1/data/nz-parl/20211118/results.json'" ] } ], "source": [ "with open(os.path.join(data_dir, \"results.json\")) as fi:\n", " res = json.load(fi)" ] }, { "cell_type": "code", "execution_count": null, "id": "33755b1a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ead245f1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0701c835", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8b2cf740", "metadata": {}, "outputs": [], "source": [ "1" ] }, { "cell_type": "code", "execution_count": null, "id": "5b1c2072", "metadata": {}, "outputs": [], "source": [ "1" ] }, { "cell_type": "code", "execution_count": null, "id": "c0746159", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9731e6c0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6ce0114a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "678f777d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "52b5e64f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6eba3eb9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e798b55a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4403d829", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c76684f6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "97cc4070", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c852d35e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "eb98df77", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 163, "id": "50b75a09", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 163, "metadata": {}, "output_type": "execute_result" } ], "source": [ "video_url = BASE_URL + \"/tab/video\"\n", "vid_resp = requests.get(video_url, headers=HEADERS)\n", "vid_resp" ] }, { "cell_type": "code", "execution_count": null, "id": "3f24ea2c", "metadata": { "scrolled": false }, "outputs": [], "source": [ "video_links = get_video_links(vid_resp)\n", "if video_links[\"youtube\"] and video_links[\"vimeo\"]:\n", " print(\"found both???\")\n", "pages_to_follow = get_more_video_links(vid_resp)\n", "for page in pages_to_follow:\n", " print(page)\n", " page_resp = requests.get(video_url + page, headers=HEADERS)\n", " if page_resp.ok:\n", " tmp = get_video_links(page_resp)\n", " video_links[\"youtube\"].extend(tmp[\"youtube\"])\n", " video_links[\"vimeo\"].extend(tmp[\"vimeo\"])\n", "video_links" ] }, { "cell_type": "code", "execution_count": null, "id": "a906c7a8", "metadata": {}, "outputs": [], "source": [ "metadata = {\"url\": orig_url, \"video_links\": video_links}" ] }, { "cell_type": "code", "execution_count": null, "id": "07fc71b9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7ca3c6f4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3f841b9f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2f7916ca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 153, "id": "0f871a50", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'https://www.parliament.nz/en/pb/hansard-debates/rhr/combined/HansD_20170803_20170803/tab/video?page=5'" ] }, "execution_count": 153, "metadata": {}, "output_type": "execute_result" } ], "source": [ "video_url + more_video_links[3]" ] }, { "cell_type": "code", "execution_count": 142, "id": "a4d200fe", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "fetching https://www.youtube.com/watch?v=d_tnT-16Guw\n", "found file /Users/mikey/data/nz-parl/20170803/d_tnT-16Guw.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=VpwS8_idNeE\n", "found file /Users/mikey/data/nz-parl/20170803/VpwS8_idNeE.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=VpwS8_idNeE\n", "found file /Users/mikey/data/nz-parl/20170803/VpwS8_idNeE.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "found file /Users/mikey/data/nz-parl/20170803/F9rgZ1NMp3U.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "found file /Users/mikey/data/nz-parl/20170803/F9rgZ1NMp3U.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "found file /Users/mikey/data/nz-parl/20170803/-u5F-DP1R3c.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "found file /Users/mikey/data/nz-parl/20170803/-u5F-DP1R3c.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "found file /Users/mikey/data/nz-parl/20170803/QriIH5RUO0Y.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "found file /Users/mikey/data/nz-parl/20170803/QriIH5RUO0Y.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "fetching https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "fetching https://www.youtube.com/watch?v=VRPck9qf7Io\n", "found file /Users/mikey/data/nz-parl/20170803/VRPck9qf7Io.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=VRPck9qf7Io\n", "found file /Users/mikey/data/nz-parl/20170803/VRPck9qf7Io.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=1epY_NZp8SA\n", "found file /Users/mikey/data/nz-parl/20170803/1epY_NZp8SA.mp4, skipping...\n", "fetching https://www.youtube.com/watch?v=1epY_NZp8SA\n", "found file /Users/mikey/data/nz-parl/20170803/1epY_NZp8SA.mp4, skipping...\n" ] } ], "source": [ "output_dir = \"/Users/mikey/data/nz-parl/20170803/\"\n", "\n", "results = []\n", "\n", "for link in video_links[\"youtube\"]:\n", " print(f\"fetching {link}\")\n", " tmp = download_youtube(link, output_dir)\n", " time.sleep(1)\n", " results.append(tmp)\n", " with open(os.path.join(output_dir, \"metadata.json\"), \"w\") as fi:\n", " json.dump(results, fi)" ] }, { "cell_type": "code", "execution_count": 144, "id": "b23ea4f7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 144, "metadata": {}, "output_type": "execute_result" } ], "source": [ "resp" ] }, { "cell_type": "code", "execution_count": null, "id": "008e9d10", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "62b0bbc5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5edbff1e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dbdf2ac0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9ddc6b23", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 73, "id": "8170d138", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'20170802'" ] }, "execution_count": 73, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "start_date = datetime.datetime.fromisoformat(\"2017-08-03\")\n", "one_day = datetime.timedelta(1)\n", "next_day = (start_date - one_day)\n", "next_day.strftime(\"%Y%m%d\")" ] }, { "cell_type": "code", "execution_count": 67, "id": "db8bc614", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "datetime.timedelta(days=1)" ] }, "execution_count": 67, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fe93b52d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8a97c300", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 6, "id": "4fcf18ee", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "resp = requests.get(BASE_URL, headers=headers,)\n", "resp" ] }, { "cell_type": "code", "execution_count": 47, "id": "0bfd417b", "metadata": {}, "outputs": [], "source": [ "with open(\"/Users/mikey/data/nz-parl/20170803/response.text\", \"w\") as fi:\n", " fi.write(resp.text)" ] }, { "cell_type": "code", "execution_count": null, "id": "52ebd5ba", "metadata": {}, "outputs": [], "source": [ "video_url = BASE_URL + \"/tab/video\"\n", "vid_resp = requests.get(video_url, headers=headers)\n", "vid_resp" ] }, { "cell_type": "code", "execution_count": 49, "id": "2f9934c9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 50, "id": "d946dca2", "metadata": {}, "outputs": [], "source": [ "video_links = []\n", "for link in BeautifulSoup(vid_resp.text, parse_only=SoupStrainer('a')):\n", " if link.has_attr('href') and \"youtube\" in link[\"href\"]:\n", " video_links.append(link['href'])" ] }, { "cell_type": "code", "execution_count": 51, "id": "c503cc4b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "20" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(video_links)" ] }, { "cell_type": "code", "execution_count": 58, "id": "b3e5a9a5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 60, "id": "0c529042", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "https://www.youtube.com/watch?v=7ld6ZD8q0yM\n", "7ld6ZD8q0yM.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=7ld6ZD8q0yM\n", "7ld6ZD8q0yM.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=yPGZ6YjTwgE\n", "yPGZ6YjTwgE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=yPGZ6YjTwgE\n", "yPGZ6YjTwgE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=d_tnT-16Guw\n", "d_tnT-16Guw.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=d_tnT-16Guw\n", "d_tnT-16Guw.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=VpwS8_idNeE\n", "VpwS8_idNeE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=VpwS8_idNeE\n", "VpwS8_idNeE.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "F9rgZ1NMp3U.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=F9rgZ1NMp3U\n", "F9rgZ1NMp3U.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "-u5F-DP1R3c.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=-u5F-DP1R3c\n", "-u5F-DP1R3c.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "QriIH5RUO0Y.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=QriIH5RUO0Y\n", "QriIH5RUO0Y.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "https://www.youtube.com/watch?v=cHhtiH_m6a0\n", "cant't find audio for {video_key}\n", "https://www.youtube.com/watch?v=VRPck9qf7Io\n", "downloading https://www.youtube.com/watch?v=VRPck9qf7Io to VRPck9qf7Io.mp4\n", "https://www.youtube.com/watch?v=VRPck9qf7Io\n", "VRPck9qf7Io.mp4 exists, skipping...\n", "https://www.youtube.com/watch?v=1epY_NZp8SA\n", "downloading https://www.youtube.com/watch?v=1epY_NZp8SA to 1epY_NZp8SA.mp4\n", "https://www.youtube.com/watch?v=1epY_NZp8SA\n", "1epY_NZp8SA.mp4 exists, skipping...\n" ] } ], "source": [ "for link in video_links:\n", " print(link)\n", " video_key = link.replace(\"https://www.youtube.com/watch?v=\", \"\")\n", " fname = video_key + \".mp4\"\n", " output_path = \"/Users/mikey/data/nz-parl/20170803\"\n", " if os.path.exists(os.path.join(output_path, fname)):\n", " print(f\"{fname} exists, skipping...\")\n", " continue\n", " yt = YouTube(link)\n", " try:\n", " s = yt.streams.filter(type=\"audio\").first()\n", " except VideoUnavailable:\n", " print(\"cant't find audio for {video_key}\")\n", " try:\n", " s = yt.streams.first()\n", " except:\n", " continue\n", " metadata[video_key] = yt.description\n", " print(f\"downloading {link} to {fname}\")\n", " s.download(output_path=output_path, filename=fname)\n", " with open(\"/Users/mikey/data/nz-parl/20170803/metadata.json\", \"w\") as fi:\n", " json.dump(metadata, fi)\n", " time.sleep(4)" ] }, { "cell_type": "code", "execution_count": 61, "id": "69dfe246", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 30, "id": "fe77196a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s.download()" ] }, { "cell_type": "code", "execution_count": 34, "id": "f10a4723", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "s" ] }, { "cell_type": "code", "execution_count": 102, "id": "b983a6bb", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 102, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 43, "id": "72b6a5e4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'2017-08-03 - Anti-Money Laundering and Countering Financing of Terrorism Amendment Bill- Third Reading - Video 12\\n\\nJono Naylor,Trevor Mallard,Matt Doocey\\n\\nHelp us caption & translate this video!\\n\\nhttp://amara.org/v/9FM2/'" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "yt.description" ] }, { "cell_type": "code", "execution_count": 44, "id": "10d3aa1c", "metadata": {}, "outputs": [], "source": [ "metadata = {}" ] }, { "cell_type": "code", "execution_count": 45, "id": "b0305ee3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/Users/mikey/data/nz-parl/20170803/7ld6ZD8q0yM.mp4'" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 46, "id": "fda7b6ea", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f9b04478", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }