{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "38bf4d0a",
"metadata": {},
"outputs": [],
"source": [
"# custom ground truth\n",
"# https://docs.google.com/document/d/1tvoaOGMSHJBlwyngR8BykqkvgYoSRRFEPKnpPa4QFqE"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8851fe11",
"metadata": {},
"outputs": [],
"source": [
"# TODO: file_name -> filename everywhere and move get_file_name to files.py"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "180199b1",
"metadata": {},
"outputs": [],
"source": [
"# # run below in ipython since this is bugged in notebooks\n",
"\n",
"# import re\n",
"# import os\n",
"# import tqdm\n",
"# from TikTokApi import TikTokApi\n",
"\n",
"# VIDEO_URLS = [\n",
"# \"https://www.tiktok.com/@alyse_jamie/video/7064380717874285870\",\n",
"# \"https://www.tiktok.com/@dezfromdarezzz/video/7055472049858350341\",\n",
"# \"https://www.tiktok.com/@hanfu94/video/7044961304007707951\",\n",
"# \"https://www.tiktok.com/@hobiwrldcore/video/7047733412068625710\",\n",
"# \"https://www.tiktok.com/@jashan_1824/video/7076739340747033858\",\n",
"# \"https://www.tiktok.com/@nxdfans/video/6988616985034771714\",\n",
"# \"https://www.tiktok.com/@reginaslay/video/7062870753846545711\",\n",
"# \"https://www.tiktok.com/@tre.markel/video/7060644104744521007\",\n",
"# \"https://www.tiktok.com/@whoisjaee/video/7051308961814842670\",\n",
"# ]\n",
"\n",
"# VIDEO_IDS = [\n",
"# ee\n",
"# for e in VIDEO_URLS\n",
"# if re.match(r\"[0-9]+\", (ee:=e.strip(\"/\").split(\"/\")[-1]))\n",
"# ]\n",
"\n",
"# DATA_DIR = \"/home/georg/notebooks/customers/speechly/inital_sample/\"\n",
"\n",
"# os.makedirs(DATA_DIR, exist_ok=True)\n",
"\n",
"# with TikTokApi() as api:\n",
"# for video_id in tqdm.tqdm(VIDEO_IDS):\n",
"# video = api.video(id=video_id)\n",
"# # TODO: add metadata\n",
"# with open(os.path.join(DATA_DIR, f\"{video_id}.mp4\"), \"wb\") as f:\n",
"# f.write(video.bytes())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3fac0e07",
"metadata": {},
"outputs": [],
"source": [
"# TODO: audio logic while keeping things compressed instead of Audio"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "4576bf87",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/7055472049858350341.mp3 already exists and will be overwritten on build\n",
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/7064380717874285870.mp3 already exists and will be overwritten on build\n",
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/7047733412068625710.mp3 already exists and will be overwritten on build\n",
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/7062870753846545711.mp3 already exists and will be overwritten on build\n",
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/7060644104744521007.mp3 already exists and will be overwritten on build\n",
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/7051308961814842670.mp3 already exists and will be overwritten on build\n",
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/rev_audio.mp3 already exists and will be overwritten on build\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.9 mins of audio\n"
]
}
],
"source": [
"import json\n",
"import numpy as np\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.utils.numbers import safe_round\n",
"\n",
"DATA_DIR = \"/home/georg/notebooks/customers/speechly/inital_sample/\"\n",
"\n",
"VIDEO_IDS = [\n",
" \"7055472049858350341\",\n",
" \"7064380717874285870\",\n",
"# \"7044961304007707951\",\n",
" \"7047733412068625710\",\n",
"# \"7076739340747033858\",\n",
"# \"6988616985034771714\",\n",
" \"7062870753846545711\",\n",
" \"7060644104744521007\",\n",
" \"7051308961814842670\",\n",
"]\n",
"\n",
"audios_rev = []\n",
"segments_meta = []\n",
"for n, video_id in enumerate(VIDEO_IDS):\n",
" audio = Audio.from_file(\n",
" os.path.join(DATA_DIR, f\"{video_id}.mp4\"),\n",
" sample_rate=44100, byte_width=2,\n",
" )\n",
" audio.to_wav(os.path.join(DATA_DIR, f\"{video_id}.wav\"))\n",
" audio.to_mp3(os.path.join(DATA_DIR, f\"{video_id}.mp3\"))\n",
" indicator_str = str(n).zfill(2)\n",
" audios_indicator = []\n",
" for c in indicator_str:\n",
" audios_indicator.append(\n",
" Audio.from_file(f\"../sanas/audio_indicators/{c}_fast.wav\", sample_rate=44100, byte_width=2)\n",
" )\n",
" audios_rev.extend(audios_indicator)\n",
" segments_meta.append({\n",
" \"segment_number\": n,\n",
" \"type\": \"indicator\",\n",
" \"duration_s\": safe_round(np.sum([a.duration_s for a in audios_indicator])),\n",
" \"indicator_str\": indicator_str,\n",
" })\n",
" audios_rev.append(audio)\n",
" segments_meta.append({\n",
" \"segment_number\": n,\n",
" \"type\": \"speech\",\n",
" \"duration_s\": safe_round(audio.duration_s),\n",
" \"orginial_audio_offset_s\": 0,\n",
" })\n",
"\n",
"audio_rev = Audio.concatenate(audios_rev)\n",
"print(round(audio_rev.duration_s / 60, 1), \"mins of audio\") \n",
"audio_rev.to_mp3(os.path.join(DATA_DIR, f\"rev_audio.mp3\"))\n",
"with open(os.path.join(DATA_DIR, f\"rev_segments.json\"), \"w\") as f:\n",
" json.dump(segments_meta, f)"
]
},
{
"cell_type": "code",
"execution_count": 60,
"id": "d210f9a0",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"audio_rev.play()"
]
},
{
"cell_type": "code",
"execution_count": 61,
"id": "85e591f5",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.tasks.music import remove_music\n",
"audio_rev_vocals = remove_music(audio_rev, wet=1.0, model_name=\"mdx_extra_q\")"
]
},
{
"cell_type": "code",
"execution_count": 62,
"id": "32c6b9ae",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"audio_rev_vocals.play()"
]
},
{
"cell_type": "code",
"execution_count": 63,
"id": "83d20da1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"output_file: /home/georg/notebooks/customers/speechly/inital_sample/rev_audio_vocals.mp3 already exists and will be overwritten on build\n"
]
}
],
"source": [
"audio_rev_vocals.to_mp3(os.path.join(DATA_DIR, f\"rev_audio_vocals.mp3\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "91fb3330",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "969c3355",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "558bd872",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "61e4cbbd",
"metadata": {},
"source": [
"## Ground truth"
]
},
{
"cell_type": "markdown",
"id": "e3900a26",
"metadata": {},
"source": [
"00\n",
"just just stop the boulder it’s not just a boulder it’s a rock\n",
"01\n",
"stop [huh] noticed how I looked you in your eyes [let's go] next time a nigga gettin' shot if you really love me fuck me like a thot if you really love me do what I say [da got that dope] when you give me head look me dead in my eye [go] nigga play and i'ma set it on fire my lil' bitch is a masterpiece\n",
"02\n",
"fuck* to be a liar [liar]\n",
"03\n",
"i'm the only provider little bad foreign bitch word to fivio niggas know the vibe that's word to five yo\n",
"beef we 'bout to resolve yo pull up on a opp do we have a problem\n",
"04\n",
"stop i’m so fucking scared right now you shut up\n",
"05\n",
"i'm tryna find something to eat if she the opp then bring her to me I don't give a fuck about her\n",
"name in these streets [sit down]"
]
},
{
"cell_type": "code",
"execution_count": 62,
"id": "e428fed6",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"audio_rev_vocals.play()"
]
},
{
"cell_type": "markdown",
"id": "239689d5",
"metadata": {},
"source": [
"Speaker 0:\n",
"00\n",
"\n",
"Speaker 1:\n",
"Just a stupid boulder.\n",
"\n",
"Speaker 2:\n",
"It’s not just a boulder, it’s a rock.\n",
"\n",
"Speaker 0:\n",
"01\n",
"\n",
"Speaker 3:\n",
"Stop. \n",
"\n",
"Speaker 4:\n",
"Huh.\n",
"\n",
"Speaker 3:\n",
"Noticed how I looked you in your eyes.\n",
"\n",
"Speaker 4:\n",
"Let's go.\n",
"\n",
"Speaker 3:\n",
"Next time a nigga gettin' shot. If you really love me fuck me like a thot. If you really love me do what I say.\n",
"\n",
"Speaker 4:\n",
"We ain't got that dope]\n",
"\n",
"Speaker 3:\n",
"When you give me head look me dead in my eye.\n",
"\n",
"Speaker 4:\n",
"Go. \n",
"\n",
"Speaker 3:\n",
"Nigga play and i'ma set it on fire. My lil' bitch is a masterpiece.\n",
"\n",
"Speaker 0:\n",
"02\n",
"\n",
"Speaker 5:\n",
"And it felt great to be a liar \n",
"\n",
"Speaker 6:\n",
"Liar.\n",
"\n",
"Speaker 0:\n",
"03\n",
"\n",
"Speaker 7:\n",
"I'm the only provider. Little bad foreign bitch word to fivio niggas know the vibe that's word to five yo\n",
"beef we 'bout to resolve yo pull up on a opp do we have a problem\n",
"\n",
"Speaker 0:\n",
"04\n",
"\n",
"Speaker 8:\n",
"Shut up! I’m so fucking scared right now. You shut up!\n",
"\n",
"Speaker 0:\n",
"05\n",
"\n",
"Speaker 9:\n",
"Let me. I'm tryna find something to eat. If she the opp then bring her to me I don't give a fuck about her\n",
"name in these streets.\n",
"\n",
"Speaker 10:\n",
"Sit down."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0dbf9f98",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cdb1d7ca",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d1ad070c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "88a9e9cc",
"metadata": {},
"source": [
"## Send stuff to rev"
]
},
{
"cell_type": "code",
"execution_count": 64,
"id": "2453089e",
"metadata": {},
"outputs": [],
"source": [
"import tqdm\n",
"import json\n",
"import uuid\n",
"from suno_utils.web.rev import (\n",
" get_auth_string, upload_file, make_order, get_finished_order, get_transcript\n",
")\n",
"from suno_utils.audio.conversion import get_audio_properties\n",
"\n",
"with open(\"/home/georg/.secrets/secrets.json\") as f:\n",
" secrets = json.load(f)\n",
" \n",
"CLIENT_API_KEY = secrets[\"REV_CLIENT_API_KEY\"]\n",
"USER_API_KEY = secrets[\"REV_USER_API_KEY\"]\n",
"AUTH_STR = get_auth_string(CLIENT_API_KEY, USER_API_KEY)"
]
},
{
"cell_type": "code",
"execution_count": 65,
"id": "b43e0b5a",
"metadata": {},
"outputs": [],
"source": [
"ORDER_REF_STR = \"tt_9\""
]
},
{
"cell_type": "code",
"execution_count": 66,
"id": "8c66f7f9",
"metadata": {},
"outputs": [],
"source": [
"audio_filepaths = [\n",
" os.path.join(DATA_DIR, f\"rev_audio.mp3\"),\n",
" os.path.join(DATA_DIR, f\"rev_audio_vocals.mp3\")\n",
"]\n",
"\n",
"media_locs = []\n",
"for audio_filepath in audio_filepaths:\n",
" media_loc = upload_file(AUTH_STR, audio_filepath, file_ref_str=audio_filepath.split(\"/\")[-1].split(\".\")[0])\n",
" media_locs.append(media_loc)\n",
" \n",
"order_items = [\n",
" {\n",
" \"media_loc\": media_loc,\n",
" \"hotwords\": [],\n",
" \"duration_s\": int(np.ceil(get_audio_properties(audio_filepath)[\"duration_s\"])),\n",
" \"uuid\": str(uuid.uuid4()),\n",
" } for media_loc, audio_filepath in zip(media_locs, audio_filepaths)\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 67,
"id": "a0ee5f4f",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"01_order_items.json\"), \"w\") as f:\n",
" json.dump(order_items, f)\n",
"# with open(os.path.join(TO_REV_DIR, \"01_order_items.json\"), \"r\") as f:\n",
"# order_items = json.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 68,
"id": "8318e46f",
"metadata": {},
"outputs": [],
"source": [
"order_number = make_order(AUTH_STR, order_items, order_ref_str=ORDER_REF_STR, add_rush=True)"
]
},
{
"cell_type": "code",
"execution_count": 70,
"id": "27549d51",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"02_order_numer.txt\"), \"w\") as f:\n",
" f.write(order_number)\n",
"# with open(os.path.join(TO_REV_DIR, \"02_order_numer.txt\")) as f:\n",
"# order_number = f.read()"
]
},
{
"cell_type": "code",
"execution_count": 47,
"id": "bfc4a570",
"metadata": {},
"outputs": [],
"source": [
"# wait until finished"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a942c30c",
"metadata": {},
"outputs": [],
"source": [
"transcript_meta = get_finished_order(AUTH_STR, order_number, blocking=True)\n",
"transcript_json = get_transcript(AUTH_STR, transcript_meta[\"rev_id\"])\n",
"with open(os.path.join(DATA_DIR, \"rev_transcript.json\"), \"w\") as f:\n",
" json.dump(transcript_json, f)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5023febd",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "8ed16479",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "2f2b9b61",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "45ee0813",
"metadata": {},
"outputs": [],
"source": [
"%pylab inline"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "aca18dec",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import cleese\n",
"\n",
"DATA_DIR = \"/home/georg/notebooks/customers/speechly/inital_sample/\"\n",
"\n",
"input_file = os.path.join(DATA_DIR, f\"rev_audio_vocals.wav\")\n",
"wave_in, sr, _ = cleese.wavRead(input_file)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "6ea9381f",
"metadata": {},
"outputs": [],
"source": [
"import pysptk\n",
"import numpy as np\n",
"\n",
"PITCH_BOUNDS = [50, 400]\n",
"PITCH_WIN = .001\n",
"\n",
"def extract_pitch(x, sr, win=PITCH_WIN, bounds=PITCH_BOUNDS): \n",
" hop_size = np.floor(sr * win)\n",
" min_f0, max_f0 = bounds\n",
" pitch = pysptk.swipe(x.astype(np.float64), fs=sr, hopsize=hop_size, min=min_f0, max=max_f0, otype=1)\n",
" times = 1000*np.arange(len(pitch))*hop_size/sr\n",
" return np.array(pitch), times"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "17f26663",
"metadata": {},
"outputs": [],
"source": [
"pitch, times = extract_pitch(wave_in, sr, win=0.005, bounds=[50, 200])"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "ad1fcaa6",
"metadata": {},
"outputs": [],
"source": [
"mean_pitch = 110.\n",
"def difference_to_cents(pitch, ref_pitch):\n",
" if pitch >0:\n",
" return -1200*np.log2(pitch/ref_pitch)\n",
" else:\n",
" return 1\n",
"bpf_times = times/1000\n",
"bpf_val = np.array([difference_to_cents(hz, mean_pitch) for hz in pitch])"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "57be864f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pitch variation 1/1\n"
]
}
],
"source": [
"config_file = \"/home/georg/tools/cleese/tutorial/configs/random_pitch_profile.py\"\n",
"bpf = np.column_stack((bpf_times,bpf_val))\n",
"wave_out,bpf_out = cleese.process(soundData=wave_in, configFile=config_file, sr=sr, BPF=bpf)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "f7c7fde6",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from IPython.display import Audio\n",
"Audio(data=wave_out, rate=sr)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0212ac90",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "31c8881f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "a3139288",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "1bb7ac67",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}