{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f712d813",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba346143",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "678239"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ids = set()\n",
    "with open(\"/data2/suno/data/harvest/youtube/pipeline/segments_meta.jsonl\") as f:\n",
    "    for line in tqdm.tqdm(f, total=997065):\n",
    "        line = line.strip()\n",
    "        if len(line) == 0:\n",
    "            continue\n",
    "        m = json.loads(line)\n",
    "        if m[\"lang_code\"].split(\"-\")[0] == \"en\":\n",
    "            ids.add(m[\"id\"])\n",
    "len(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "769031c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# processing time: 67/199 [57:00:51<111:26:29\n",
    "# s4cmd ls s3://suno-data/datasets/harvest/youtube/dataset/en | wc -l\n",
    "# 551699"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d2b090bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31.7171380590028"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "170 * (678239-551699) / 678239"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "815f9097",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60513e5b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24f46498",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fe5d466",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "738e3242",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ffmpeg -y -i IqJnfh42LVA.m4a -c:a libopus -b:a 32K -t 5 -ss 5 tmp/test_1.opus -c:a libopus -b:a 32K -t 5 -ss 10 tmp/test_2.opus -c:a libopus -b:a 32K -t 5 -ss 15 tmp/test_3.opus -c:a libopus -b:a 32K -t 5 -ss 55 tmp/test_4.opus\n"
     ]
    }
   ],
   "source": [
    "convert_cmd_lines = [\n",
    "    \"ffmpeg -y -i IqJnfh42LVA.m4a\",\n",
    "    \"-c:a libopus -b:a 32K -t 5 -ss 5 tmp/test_1.opus\",\n",
    "    \"-c:a libopus -b:a 32K -t 5 -ss 10 tmp/test_2.opus\",\n",
    "    \"-c:a libopus -b:a 32K -t 5 -ss 15 tmp/test_3.opus\",\n",
    "    \"-c:a libopus -b:a 32K -t 5 -ss 55 tmp/test_4.opus\",\n",
    "]\n",
    "print(\" \".join(convert_cmd_lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b07a0e83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[default]\r\n",
      "aws_access_key_id = AKIA2V4GXGDKEG2DGA5L\r\n",
      "aws_secret_access_key = 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\r\n"
     ]
    }
   ],
   "source": [
    "!cat ~/.aws/credentials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a308f0ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00_get_diverse_ids.ipynb  lid.176.bin\t      test_1.opus\r\n",
      "01_get_data.ipynb\t  old_pipeline.ipynb  test_2.opus\r\n",
      "02_prep_corpus.ipynb\t  test.mp3\t      tmp\r\n",
      "IqJnfh42LVA.m4a\t\t  test.opus\t      verified_vtt_fns.json\r\n",
      "Untitled.ipynb\t\t  test.wav\r\n",
      "Untitled1.ipynb\t\t  test_0.opus\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "060948a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "s3 = boto3.resource(\"s3\")\n",
    "BUCKET = \"test\"\n",
    "\n",
    "s3.Bucket(BUCKET).upload_file(\"your/local/file\", \"dump/file\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a35b170",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3cmd ls s3://suno-data/datasets/harvest/youtube/dataset/en/uuid/segment_uuid.opus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3d742874",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 194 µs, sys: 4.13 ms, total: 4.32 ms\n",
      "Wall time: 440 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import subprocess\n",
    "out = subprocess.call(\n",
    "    \" \".join(convert_cmd_lines).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n",
    ")\n",
    "assert(out == 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8dbeec2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 807 µs, sys: 4.31 ms, total: 5.11 ms\n",
      "Wall time: 1.74 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import subprocess\n",
    "out = subprocess.call(\n",
    "    \"ffmpeg -y -i IqJnfh42LVA.m4a -c:a libopus -b:a 32K tmp/test.opus\".split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n",
    ")\n",
    "assert(out == 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "61a6ccfb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ffmpeg version 4.4.2-0ubuntu0.22.04.1 Copyright (c) 2000-2021 the FFmpeg developers\n",
      "  built with gcc 11 (Ubuntu 11.2.0-19ubuntu1)\n",
      "  configuration: --prefix=/usr --extra-version=0ubuntu0.22.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libdav1d --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librabbitmq --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-pocketsphinx --enable-librsvg --enable-libmfx --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared\n",
      "  libavutil      56. 70.100 / 56. 70.100\n",
      "  libavcodec     58.134.100 / 58.134.100\n",
      "  libavformat    58. 76.100 / 58. 76.100\n",
      "  libavdevice    58. 13.100 / 58. 13.100\n",
      "  libavfilter     7.110.100 /  7.110.100\n",
      "  libswscale      5.  9.100 /  5.  9.100\n",
      "  libswresample   3.  9.100 /  3.  9.100\n",
      "  libpostproc    55.  9.100 / 55.  9.100\n",
      "Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'IqJnfh42LVA.m4a':\n",
      "  Metadata:\n",
      "    major_brand     : isom\n",
      "    minor_version   : 512\n",
      "    compatible_brands: isomiso2mp41\n",
      "    encoder         : Lavf58.76.100\n",
      "  Duration: 00:01:23.87, start: 0.000000, bitrate: 42 kb/s\n",
      "  Stream #0:0(eng): Audio: aac (HE-AAC) (mp4a / 0x6134706D), 44100 Hz, stereo, fltp, 42 kb/s (default)\n",
      "    Metadata:\n",
      "      handler_name    : ISO Media file produced by Google Inc. Created on: 10/22/2018.\n",
      "      vendor_id       : [0][0][0][0]\n",
      "Stream mapping:\n",
      "  Stream #0:0 -> #0:0 (aac (native) -> opus (libopus))\n",
      "  Stream #0:0 -> #1:0 (aac (native) -> opus (libopus))\n",
      "  Stream #0:0 -> #2:0 (aac (native) -> opus (libopus))\n",
      "Press [q] to stop, [?] for help\n",
      "Output #0, opus, to 'tmp/test_1.opus':\n",
      "  Metadata:\n",
      "    major_brand     : isom\n",
      "    minor_version   : 512\n",
      "    compatible_brands: isomiso2mp41\n",
      "    encoder         : Lavf58.76.100\n",
      "  Stream #0:0(eng): Audio: opus, 48000 Hz, stereo, flt, 32 kb/s (default)\n",
      "    Metadata:\n",
      "      handler_name    : ISO Media file produced by Google Inc. Created on: 10/22/2018.\n",
      "      vendor_id       : [0][0][0][0]\n",
      "      encoder         : Lavc58.134.100 libopus\n",
      "      major_brand     : isom\n",
      "      minor_version   : 512\n",
      "      compatible_brands: isomiso2mp41\n",
      "Output #1, opus, to 'tmp/test_2.opus':\n",
      "  Metadata:\n",
      "    major_brand     : isom\n",
      "    minor_version   : 512\n",
      "    compatible_brands: isomiso2mp41\n",
      "    encoder         : Lavf58.76.100\n",
      "  Stream #1:0(eng): Audio: opus, 48000 Hz, stereo, flt, 32 kb/s (default)\n",
      "    Metadata:\n",
      "      handler_name    : ISO Media file produced by Google Inc. Created on: 10/22/2018.\n",
      "      vendor_id       : [0][0][0][0]\n",
      "      encoder         : Lavc58.134.100 libopus\n",
      "      major_brand     : isom\n",
      "      minor_version   : 512\n",
      "      compatible_brands: isomiso2mp41\n",
      "Output #2, opus, to 'tmp/test_3.opus':\n",
      "  Metadata:\n",
      "    major_brand     : isom\n",
      "    minor_version   : 512\n",
      "    compatible_brands: isomiso2mp41\n",
      "    encoder         : Lavf58.76.100\n",
      "  Stream #2:0(eng): Audio: opus, 48000 Hz, stereo, flt, 32 kb/s (default)\n",
      "    Metadata:\n",
      "      handler_name    : ISO Media file produced by Google Inc. Created on: 10/22/2018.\n",
      "      vendor_id       : [0][0][0][0]\n",
      "      encoder         : Lavc58.134.100 libopus\n",
      "      major_brand     : isom\n",
      "      minor_version   : 512\n",
      "      compatible_brands: isomiso2mp41\n",
      "size=      18kB time=00:00:04.98 bitrate=  29.7kbits/s speed=  28x    \n",
      "video:0kB audio:18kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 3.196118%\n"
     ]
    }
   ],
   "source": [
    "!ffmpeg -y -i IqJnfh42LVA.m4a -c:a libopus -b:a 32K -t 5 -ss 5 tmp/test_1.opus -c:a libopus -b:a 32K -t 5 -ss 10 tmp/test_2.opus -c:a libopus -b:a 32K -t 5 -ss 15 tmp/test_3.opus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb4d67c2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1f26865",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}