{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "b133bd18", "metadata": {}, "outputs": [], "source": [ "def get_data(identifier):\n", " ia.download(identifier,\n", " formats=[\"SubRip\", \"VBR MP3\", \"MP3\", \"Web Video Text Tracks\", \"Closed Caption Text\"],\n", " destdir=save_directory,\n", " # Very import to set this. tf.io.gfile uses mtime in\n", " # nanoseconds, while archive.org uses mtime in seconds\n", " # (as far as I can tell). I could convert the\n", " # nanoseconds to seconds, of course, but don't want to\n", " # make an error.\n", " ignore_existing=True,\n", " # tf.io.gfile does not expose any functionality like os.utime\n", " no_change_timestamp=True,\n", " ignore_errors=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "04b7f309", "metadata": {}, "outputs": [], "source": [ "ids = []\n", "with gzip.open(metadata_file, \"rt\") as fh:\n", " for line in fh:\n", " ids.append(json.loads(line)[\"identifier\"])\n", " \n", "get_data(ids[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "82f30abd", "metadata": {}, "outputs": [], "source": [ "def download_metadata(query, save_file):\n", " search = ia.search_items(query)\n", " all_results = list(tqdm(search.iter_as_results()))\n", "\n", " def get_metadata(result: dict):\n", " item = ia.get_item(result[\"identifier\"],\n", " archive_session=search.session)\n", " metadata = item.item_metadata\n", " metadata[\"identifier\"] = result[\"identifier\"]\n", " return metadata\n", "\n", " with ThreadPoolExecutor(15) as executor:\n", " metadata_list = list(tqdm(executor.map(get_metadata, all_results), total=len(all_results)))\n", " with gzip.open(save_file, \"wt\") as fh:\n", " for metadata in metadata_list:\n", " # Don't use jsonl, use... \n", " json.dump(metadata, fh)\n", " fh.write(\"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "id": "ef31f369", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 2, "id": "91b99132", "metadata": {}, "outputs": [], "source": [ "LICENSE_WHITELIST = \"\"\"(\\\n", "licenseurl:*creativecommons.org\\/licenses\\/by-sa\\/4.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by-sa\\/3.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by-sa\\/2.5* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by-sa\\/2.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by-sa\\/1.0* OR \\\n", "licenseurl:*creativecommons.org\\/publicdomain\\/zero\\/1.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by\\/4.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by\\/3.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by\\/2.5* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by\\/2.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/by\\/1.0* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/publicdomain* OR \\\n", "licenseurl:*creativecommons.org\\/publicdomain\\/mark\\/1.0* OR \\\n", "licenseurl:*usa.gov\\/government-works* OR \\\n", "licenseurl:*creativecommons.org\\/licenses\\/cc0\\/3.0* \\\n", ")\"\"\"\n", "\n", "QUERIES = dict(\n", " # TODO: Consider adding AND NOT format:ASR to disclude speech recognition-based labels.\n", " #ALL_CAPTIONED_DATA=f\"{LICENSE_WHITELIST} AND (mediatype:audio OR mediatype:movies) AND (closed_captioning:yes OR format:SubRip OR format:\\\"Web Video Text Tracks\\\") AND (NOT access-restricted-item:TRUE)\",\n", " # NON_CAPTIONED_DATA_WITH_TEXT=f\"{LICENSE_WHITELIST} AND (format:DjVuTXT AND format:MP3 AND NOT format:SubRip) AND NOT (subject:'librivox')\",\n", " CC_BY_SA_EXPANDED_LICENSES_FILTERED_ACCESS=f\"{LICENSE_WHITELIST} AND (mediatype:audio OR mediatype:movies) AND (closed_captioning:yes OR format:SubRip OR format:\\\"Web Video Text Tracks\\\") AND (NOT access-restricted-item:TRUE)\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "392bae1d", "metadata": {}, "outputs": [], "source": [ "for key, query in QUERIES.items():\n", " print(query)\n", " print(f\"Dumping metadata for {key}\")\n", " save_file = key + \".jsonl.gz\"\n", " download_metadata(query, save_file)\n", " download_data(save_file, f\"gs://the-peoples-speech-west-europe/archive_org/Mar_7_2021/{key}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3facdf02", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d48db228", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "1e08367f", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 4, "id": "a7d69582", "metadata": {}, "outputs": [], "source": [ "import internetarchive as ia" ] }, { "cell_type": "code", "execution_count": 39, "id": "a4099000", "metadata": {}, "outputs": [], "source": [ "LICENSE_WHITELIST = \"\"\"(\\\n", " licenseurl:*creativecommons.org\\/publicdomain\\/zero\\/1.0* OR \\\n", " licenseurl:*creativecommons.org\\/licenses\\/by\\/4.0* OR \\\n", " licenseurl:*creativecommons.org\\/licenses\\/by\\/3.0* OR \\\n", " licenseurl:*creativecommons.org\\/licenses\\/by\\/2.5* OR \\\n", " licenseurl:*creativecommons.org\\/licenses\\/by\\/2.0* OR \\\n", " licenseurl:*creativecommons.org\\/licenses\\/by\\/1.0* OR \\\n", " licenseurl:*creativecommons.org\\/licenses\\/publicdomain* OR \\\n", " licenseurl:*creativecommons.org\\/publicdomain\\/mark\\/1.0* OR \\\n", " licenseurl:*usa.gov\\/government-works* OR \\\n", " licenseurl:*creativecommons.org\\/licenses\\/cc0\\/3.0* \\\n", ")\"\"\"\n", "\n", "\n", "QUERY_STR = LICENSE_WHITELIST + \"\"\" AND \\\n", " (mediatype:audio OR mediatype:movies) AND \\\n", " (closed_captioning:yes OR format:SubRip OR format:\\\"Web Video Text Tracks\\\") AND \\\n", " (NOT access-restricted-item:TRUE) \\\n", "\"\"\"\n", "# format \"WEB VIDEO TEXT TRACKS\" exclude name ends in - .es.vtt\n", "# format close captions cc5.txt\n", "# format sub rip cc5.srt asr.srt" ] }, { "cell_type": "code", "execution_count": 40, "id": "951f88d3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "51201\n" ] } ], "source": [ "search = ia.search_items(QUERY_STR)\n", "print(search.num_found)" ] }, { "cell_type": "code", "execution_count": 41, "id": "7f063797", "metadata": {}, "outputs": [], "source": [ "data = []\n", "for item in search.iter_as_results():\n", " data.append(item)\n", " if len(data) >= 100:\n", " break" ] }, { "cell_type": "code", "execution_count": 48, "id": "59e63a4f", "metadata": {}, "outputs": [], "source": [ "n = 27\n", "_id = data[n][\"identifier\"]\n", "item = ia.get_item(_id, archive_session=search.session)\n", "metadata = item.item_metadata\n", "assert(metadata['metadata']['identifier'] == _id)" ] }, { "cell_type": "code", "execution_count": 49, "id": "4029d687", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'identifier': 'gov.uscourts.ca1.13758',\n", " 'collection': 'uscourtsoralarguments',\n", " 'contributor': 'Free Law Project',\n", " 'court': 'ca1',\n", " 'description': 'This item represents an oral argument audio file as scraped from a U.S. Government website by Free Law Project.',\n", " 'language': 'eng',\n", " 'licenseurl': 'https://www.usa.gov/government-works',\n", " 'mediatype': 'audio',\n", " 'scanner': 'Internet Archive Python library 1.8.1',\n", " 'source_url': 'https://www.courtlistener.com/audio/14195/students-for-fair-admissions-v-president-and-fellows/',\n", " 'title': 'Students for Fair Admissions v. President and Fellows',\n", " 'publicdate': '2018-08-29 01:26:58',\n", " 'uploader': 'recapinfo@gmail.com',\n", " 'addeddate': '2018-08-29 01:26:58',\n", " 'backup_location': 'ia906801_11'}" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metadata['metadata']" ] }, { "cell_type": "code", "execution_count": null, "id": "d7fa3c3a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "485b21bb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 18, "id": "e5005b01", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'00BienvenidoALaRevolucinDeLaImpresinEn3D'" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data[0][\"identifier\"]" ] }, { "cell_type": "code", "execution_count": 15, "id": "e00fe796", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'identifier': '00BienvenidoALaRevolucinDeLaImpresinEn3D',\n", " 'mediatype': 'movies',\n", " 'collection': ['opensource_movies', 'community'],\n", " 'creator': 'AT86',\n", " 'date': '2018-09-17',\n", " 'description': '3d',\n", " 'language': 'eng',\n", " 'licenseurl': 'http://creativecommons.org/publicdomain/mark/1.0/',\n", " 'scanner': 'Internet Archive HTML5 Uploader 1.6.3',\n", " 'subject': '3d',\n", " 'title': '00 Bienvenido A La Revolución De La Impresión En 3 D',\n", " 'publicdate': '2018-09-18 00:15:08',\n", " 'uploader': '90.g.hr.56@gmail.com',\n", " 'addeddate': '2018-09-18 00:15:08',\n", " 'curation': '[curator]validator@archive.org[/curator][date]20180918001620[/date][comment]checked for malware[/comment]',\n", " 'backup_location': 'ia906802_1'}" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "assert(data[0][\"identifier\"] == item.metadata['identifier'])" ] }, { "cell_type": "code", "execution_count": 16, "id": "d71e7143", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'created': 1645806989,\n", " 'd1': 'ia902800.us.archive.org',\n", " 'd2': 'ia802800.us.archive.org',\n", " 'dir': '/5/items/00BienvenidoALaRevolucinDeLaImpresinEn3D',\n", " 'files': [{'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000001.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229929',\n", " 'size': '4321',\n", " 'md5': 'f74426088ab8de5894d3d2622413f631',\n", " 'crc32': '35a5bea6',\n", " 'sha1': '7dbd1e531abe092cae1059f490fc0053748957cb'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000057.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229937',\n", " 'size': '7200',\n", " 'md5': 'fa9bbac9eaea68ff9619a3a0f6dac13c',\n", " 'crc32': '9968aae7',\n", " 'sha1': '7b8b7cac709cc2d0d8c0772d3d9c3f85da358353'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000087.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229945',\n", " 'size': '3951',\n", " 'md5': '263e0cda61402dd74646b94dcc9d0da8',\n", " 'crc32': '536c28b2',\n", " 'sha1': 'c4a829ebef0e7187e0a7a44d594ed96438e0cb62'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000117.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229953',\n", " 'size': '5541',\n", " 'md5': '20631a5719b3dad2ef3a29eb77ccb4b8',\n", " 'crc32': 'd73376e3',\n", " 'sha1': 'de4d1762aa27f5c3ee9a4d11fccfb136536f18d0'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000147.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229961',\n", " 'size': '7188',\n", " 'md5': 'c00696e59b991c64acc97753ccfeedfb',\n", " 'crc32': '79aab858',\n", " 'sha1': 'c7eb706a1888646bad0e1b915ee28eb9c4800dff'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000177.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229969',\n", " 'size': '4015',\n", " 'md5': '76615c8f2e4fd8c4b9f3ef72afb63454',\n", " 'crc32': '38699e07',\n", " 'sha1': '193d772803a4d9e053a8965cf21c9aba8851c283'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000207.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229977',\n", " 'size': '7518',\n", " 'md5': '4bafc0f19555fcb564b0d2cafdffd249',\n", " 'crc32': '1743bfe8',\n", " 'sha1': 'cecf2eeef70e233d90db961469f4ebe5ff0e8082'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000237.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229985',\n", " 'size': '6254',\n", " 'md5': '98e4d9bd5195faa845008657b4921822',\n", " 'crc32': '61e02120',\n", " 'sha1': 'd715567d634867bbd2dff5e6dff5f794fbbd8d5f'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000267.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537229993',\n", " 'size': '3956',\n", " 'md5': '15ca124a52f45c248ac52209d48964fc',\n", " 'crc32': 'efcbb158',\n", " 'sha1': 'f5d76416d9aed799dc56909bc667a5dec0f28b93'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000297.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537230001',\n", " 'size': '4157',\n", " 'md5': '2eb7a3581bd957be83fd13ee44220a39',\n", " 'crc32': '12721acc',\n", " 'sha1': '16e3b5ca26ae8ebf0e17223956d99b12c03904ce'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000327.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537230008',\n", " 'size': '3929',\n", " 'md5': '128b4e191b7af1439ef42e1e38d9a1e2',\n", " 'crc32': '1314fcb0',\n", " 'sha1': '7a7b6f17be5d101700723ae0f1fd140dc19e9ab5'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D.thumbs/00_Bienvenido a la revolución de la impresión en 3D_000357.jpg',\n", " 'source': 'derivative',\n", " 'format': 'Thumbnail',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537230015',\n", " 'size': '5513',\n", " 'md5': '640e974ffda6a86374b9ffd6263eeded',\n", " 'crc32': 'ec6ce4a7',\n", " 'sha1': 'c25535fc9ea4e886b8b3de5dc064d5a9b1f7128c'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D_archive.torrent',\n", " 'source': 'metadata',\n", " 'btih': 'c9877fcd5ca4574625e05c4fc8c2c67bf5d52bf6',\n", " 'mtime': '1537230521',\n", " 'size': '3973',\n", " 'md5': 'd0da4f635f0d53dd3df40a488991dd68',\n", " 'crc32': '6b3f2687',\n", " 'sha1': '31da5e4b18bea462d621adac2d4f0ed2c770ef5f',\n", " 'format': 'Archive BitTorrent'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D_files.xml',\n", " 'source': 'original',\n", " 'format': 'Metadata',\n", " 'md5': '156c8f8904b284a9c832a9fbdb3c4cd8'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D_meta.sqlite',\n", " 'source': 'original',\n", " 'mtime': '1537229896',\n", " 'size': '14336',\n", " 'format': 'Metadata',\n", " 'md5': 'ea8018c34237224748100de36b2547a5',\n", " 'crc32': '57c6f8c8',\n", " 'sha1': '56ad1998bd7d7b4606bc72ad600160065c0ef293'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D_meta.xml',\n", " 'source': 'original',\n", " 'mtime': '1613076782',\n", " 'size': '900',\n", " 'format': 'Metadata',\n", " 'md5': 'e9f9edee2672b06e7ffbf214e90ecbbf',\n", " 'crc32': '94d46509',\n", " 'sha1': 'd7bdf4ac363b0db7a9ce809f96e503911b0b1f05'},\n", " {'name': '00BienvenidoALaRevolucinDeLaImpresinEn3D_reviews.xml',\n", " 'source': 'original',\n", " 'mtime': '1561425081',\n", " 'size': '201',\n", " 'format': 'Metadata',\n", " 'md5': '35138f7c92fb08789f5a84f13fe0dd7b',\n", " 'crc32': '5b5324b2',\n", " 'sha1': '9feadf58d5255b7f099122f4a9b20b1cd0c6eee5'},\n", " {'name': '00_Bienvenido a la revolución de la impresión en 3D.es.vtt',\n", " 'source': 'original',\n", " 'mtime': '1537229706',\n", " 'size': '8968',\n", " 'md5': 'fe0de11dc1b6f77da381192fb773b5f3',\n", " 'crc32': '7efa68c4',\n", " 'sha1': '183a02f4d58620dc8acee79feac5d2cf782644f7',\n", " 'format': 'Web Video Text Tracks'},\n", " {'name': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'source': 'original',\n", " 'mtime': '1537229886',\n", " 'size': '19575558',\n", " 'md5': 'dd45df23e2f54dc3bc537d5b09c8230d',\n", " 'crc32': 'f95987f1',\n", " 'sha1': 'a42420e4d7ba4b89ccc5b465baeeba3491fe0f88',\n", " 'format': 'MPEG4',\n", " 'length': '386.03',\n", " 'height': '540',\n", " 'width': '960'},\n", " {'name': '00_Bienvenido a la revolución de la impresión en 3D.ogv',\n", " 'source': 'derivative',\n", " 'format': 'Ogg Video',\n", " 'original': '00_Bienvenido a la revolución de la impresión en 3D.mp4',\n", " 'mtime': '1537230518',\n", " 'size': '28852925',\n", " 'md5': '07b165d55b6d5de792db9e96dbf3b328',\n", " 'crc32': 'f5001860',\n", " 'sha1': 'db5670ab84b5608ee365c040ac120470ce583e85',\n", " 'length': '386.03',\n", " 'height': '300',\n", " 'width': '533'},\n", " {'name': '__ia_thumb.jpg',\n", " 'source': 'original',\n", " 'mtime': '1613076782',\n", " 'size': '8468',\n", " 'md5': '95dfd637230a41ef4fbb9a17d17519df',\n", " 'crc32': 'cd8b9fb6',\n", " 'sha1': 'ceb6c5bee9b3683140298734766c8709a03762d9',\n", " 'format': 'Item Tile',\n", " 'rotation': '0'}],\n", " 'files_count': 21,\n", " 'item_last_updated': 1613076782,\n", " 'item_size': 48528872,\n", " 'metadata': {'identifier': '00BienvenidoALaRevolucinDeLaImpresinEn3D',\n", " 'mediatype': 'movies',\n", " 'collection': ['opensource_movies', 'community'],\n", " 'creator': 'AT86',\n", " 'date': '2018-09-17',\n", " 'description': '3d',\n", " 'language': 'eng',\n", " 'licenseurl': 'http://creativecommons.org/publicdomain/mark/1.0/',\n", " 'scanner': 'Internet Archive HTML5 Uploader 1.6.3',\n", " 'subject': '3d',\n", " 'title': '00 Bienvenido A La Revolución De La Impresión En 3 D',\n", " 'publicdate': '2018-09-18 00:15:08',\n", " 'uploader': '90.g.hr.56@gmail.com',\n", " 'addeddate': '2018-09-18 00:15:08',\n", " 'curation': '[curator]validator@archive.org[/curator][date]20180918001620[/date][comment]checked for malware[/comment]',\n", " 'backup_location': 'ia906802_1'},\n", " 'reviews': [],\n", " 'server': 'ia902800.us.archive.org',\n", " 'uniq': 1332013786,\n", " 'workable_servers': ['ia902800.us.archive.org', 'ia802800.us.archive.org'],\n", " 'identifier': '00BienvenidoALaRevolucinDeLaImpresinEn3D'}" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "item.item_metadata" ] }, { "cell_type": "code", "execution_count": 14, "id": "25ef04dc", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "IdentifierListAsItems(['opensource_movies', 'community'])" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "item.collection" ] }, { "cell_type": "code", "execution_count": null, "id": "a4abeca2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "12430f6c", "metadata": {}, "outputs": [], "source": [ "def get_metadata(result):\n", " \n", " metadata = item.item_metadata\n", " metadata[\"identifier\"] = result[\"identifier\"]\n", " return metadata" ] }, { "cell_type": "code", "execution_count": 8, "id": "2cbc2e09", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'identifier': '04TheMostDangerousGame'}" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data[10]" ] }, { "cell_type": "code", "execution_count": null, "id": "9e25ea37", "metadata": {}, "outputs": [], "source": [ "all_results = list(tqdm(search.iter_as_results()))\n", "\n", " def get_metadata(result: dict):\n", " item = ia.get_item(result[\"identifier\"],\n", " archive_session=search.session)\n", " metadata = item.item_metadata\n", " metadata[\"identifier\"] = result[\"identifier\"]\n", " return metadata\n", "\n", " with ThreadPoolExecutor(15) as executor:\n", " metadata_list = list(tqdm(executor.map(get_metadata, all_results), total=len(all_results)))\n", " with gzip.open(save_file, \"wt\") as fh:\n", " for metadata in metadata_list:\n", " # Don't use jsonl, use... \n", " json.dump(metadata, fh)\n", " fh.write(\"\\n\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a9961e38", "metadata": {}, "outputs": [], "source": [ " with gzip.open(save_file, \"wt\") as fh:\n", " for metadata in metadata_list:\n", " # Don't use jsonl, use... \n", " json.dump(metadata, fh)\n", " fh.write(\"\\n\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "99b9b1e8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c2e3feaf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7f7e28d3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fa1dd349", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }