{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "f4d30718-df4d-427e-b465-652d4aea7469",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import math\n",
"import tempfile\n",
"\n",
"import numpy as np\n",
"import torch\n",
"import tqdm\n",
"\n",
"# from .generation import (\n",
"# preload_models as preload_gpt_models,\n",
"# load_model as load_gpt_model,\n",
"# generate_semantic,\n",
"# generate_coarse,\n",
"# )\n",
"# from ../asr import whisper\n",
"from suno_utils.tasks.asr.whisper import transcribe\n",
"# from ..encodec import (\n",
"# preload_models as preload_encodec_models,\n",
"# encode as codec_encode,\n",
"# decode as codec_decode,\n",
"# )\n",
"# from ..mert import (\n",
"# preload_models as preload_mert_models,\n",
"# encode as embed_encode,\n",
"# )\n",
"# from ...audio.conversion import Audio\n",
"# from ...utils.s3 import _download_s3_file\n",
"# from ...web.harvest import get_filename"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "672180ae-8536-4c9d-81ea-1fcf506944ec",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from suno_utils.worker import video_seewav\n",
"from suno_utils.audio import Audio\n",
"mp3_path = \"/Users/anjaleenarenthiren/Downloads/witch.mp3\"\n",
"png_path = \"/Users/anjaleenarenthiren/Downloads/witch.png\"\n",
"a = Audio.from_file(filepath=mp3_path)\n",
"a.play()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "37415e6e-ff69-4e1a-a6a6-c4b12587cd65",
"metadata": {},
"outputs": [],
"source": [
"history_text = transcribe(a)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "51438fb8-bbe9-4646-ad21-2db92a413f73",
"metadata": {},
"outputs": [],
"source": [
"segments = history_text['segments']\n",
"transcription = []\n",
"for s in segments:\n",
" for w in s['words']:\n",
" transcription.append(w)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "5bc99b96-fa50-47be-8f18-d3efd7ae199d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['I told the witch doctor, I was in love with you', 'I told the witch doctor, I was in love with you', 'And then the witch doctor, he told me what to do', 'He told me:', '', 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', '']\n",
"['I told the witch doctor, I was in love with you', 'I told the witch doctor, I was in love with you', 'And then the witch doctor, he told me what to do', 'He told me:', '', 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', '']\n"
]
}
],
"source": [
"prompt_raw = (\"\"\"I told the witch doctor, I was in love with you\n",
"I told the witch doctor, I was in love with you\n",
"And then the witch doctor, he told me what to do\n",
"He told me:\n",
"\n",
"Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang\n",
"Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang\n",
"\"\"\")\n",
"prompt = prompt_raw.split(\"\\n\")\n",
"prompt_lines = prompt_raw.split(\"\\n\")\n",
"print (prompt)\n",
"print (prompt_lines)\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ee0be1d4-1cc1-4bf6-aaae-c58cb9514e4d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
">>>> {'id': 0, 'seek': 0, 'start': 0.52, 'end': 5.96, 'text': ' I told the witch doctor, I was in love with you', 'tokens': [50363, 314, 1297, 262, 16365, 6253, 11, 314, 373, 287, 1842, 351, 345, 50613], 'temperature': 0.0, 'avg_logprob': -0.3545205489448879, 'compression_ratio': 1.8932038834951457, 'no_speech_prob': 0.48473232984542847, 'words': [{'word': ' I', 'start': 0.52, 'end': 1.12, 'probability': 0.4590614438056946}, {'word': ' told', 'start': 1.12, 'end': 1.44, 'probability': 0.747996985912323}, {'word': ' the', 'start': 1.44, 'end': 1.64, 'probability': 0.8343585729598999}, {'word': ' witch', 'start': 1.64, 'end': 1.86, 'probability': 0.29270243644714355}, {'word': ' doctor,', 'start': 1.86, 'end': 2.16, 'probability': 0.9103218913078308}, {'word': ' I', 'start': 2.66, 'end': 3.42, 'probability': 0.9580233693122864}, {'word': ' was', 'start': 3.42, 'end': 3.7, 'probability': 0.8647823929786682}, {'word': ' in', 'start': 3.7, 'end': 3.94, 'probability': 0.6030638813972473}, {'word': ' love', 'start': 3.94, 'end': 4.16, 'probability': 0.9732234477996826}, {'word': ' with', 'start': 4.16, 'end': 4.38, 'probability': 0.9944580793380737}, {'word': ' you', 'start': 4.38, 'end': 4.58, 'probability': 0.9525207281112671}, {'word': ' I', 'start': 4.58, 'end': 5.1, 'probability': 0.4953804016113281}, {'word': ' told', 'start': 5.1, 'end': 5.96, 'probability': 0.9938238859176636}]}\n",
">>>> {'id': 1, 'seek': 0, 'start': 5.96, 'end': 11.0, 'text': ' I told the witch doctor, I was in love with you', 'tokens': [50613, 314, 1297, 262, 16365, 6253, 11, 314, 373, 287, 1842, 351, 345, 50863], 'temperature': 0.0, 'avg_logprob': -0.3545205489448879, 'compression_ratio': 1.8932038834951457, 'no_speech_prob': 0.48473232984542847, 'words': [{'word': ' the', 'start': 5.96, 'end': 6.2, 'probability': 0.9971298575401306}, {'word': ' witch', 'start': 6.2, 'end': 6.42, 'probability': 0.987332284450531}, {'word': ' doctor,', 'start': 6.42, 'end': 6.86, 'probability': 0.998045802116394}, {'word': ' I', 'start': 7.52, 'end': 8.0, 'probability': 0.9941814541816711}, {'word': ' was', 'start': 8.0, 'end': 8.28, 'probability': 0.9959487915039062}, {'word': ' in', 'start': 8.28, 'end': 8.5, 'probability': 0.9990255832672119}, {'word': ' love', 'start': 8.5, 'end': 8.74, 'probability': 0.999404788017273}, {'word': ' with', 'start': 8.74, 'end': 8.94, 'probability': 0.9991326928138733}, {'word': ' you', 'start': 8.94, 'end': 9.26, 'probability': 0.9954517483711243}, {'word': ' And', 'start': 9.26, 'end': 10.26, 'probability': 0.8809637427330017}, {'word': ' then', 'start': 10.26, 'end': 10.52, 'probability': 0.9808691740036011}, {'word': ' the', 'start': 10.52, 'end': 10.76, 'probability': 0.9614416360855103}, {'word': ' witch', 'start': 10.76, 'end': 11.0, 'probability': 0.9872244596481323}]}\n",
">>>> {'id': 2, 'seek': 0, 'start': 11.0, 'end': 19.7, 'text': ' And then the witch doctor, he told me what to do', 'tokens': [50863, 843, 788, 262, 16365, 6253, 11, 339, 1297, 502, 644, 284, 466, 51063], 'temperature': 0.0, 'avg_logprob': -0.3545205489448879, 'compression_ratio': 1.8932038834951457, 'no_speech_prob': 0.48473232984542847, 'words': [{'word': ' doctor,', 'start': 11.0, 'end': 11.4, 'probability': 0.9967849254608154}, {'word': ' he', 'start': 11.84, 'end': 12.48, 'probability': 0.8257579803466797}, {'word': ' told', 'start': 12.48, 'end': 12.72, 'probability': 0.988128125667572}, {'word': ' me', 'start': 12.72, 'end': 12.96, 'probability': 0.9966263771057129}, {'word': ' what', 'start': 12.96, 'end': 13.26, 'probability': 0.926365077495575}, {'word': ' to', 'start': 13.26, 'end': 13.5, 'probability': 0.5824445486068726}, {'word': ' do', 'start': 13.5, 'end': 14.04, 'probability': 0.9856362342834473}, {'word': ' He', 'start': 14.04, 'end': 14.26, 'probability': 0.38266023993492126}, {'word': ' told', 'start': 14.26, 'end': 15.0, 'probability': 0.9868391156196594}, {'word': ' me', 'start': 15.0, 'end': 15.32, 'probability': 0.9992640614509583}, {'word': ' We', 'start': 15.32, 'end': 16.8, 'probability': 0.020928941667079926}, {'word': ' One,', 'start': 16.8, 'end': 19.7, 'probability': 0.012912625446915627}]}\n",
">>>> {'id': 3, 'seek': 0, 'start': 20.98, 'end': 22.8, 'text': ' He told me', 'tokens': [51063, 679, 1297, 502, 51163], 'temperature': 0.0, 'avg_logprob': -0.3545205489448879, 'compression_ratio': 1.8932038834951457, 'no_speech_prob': 0.48473232984542847, 'words': [{'word': ' two,', 'start': 20.98, 'end': 21.04, 'probability': 0.21501576900482178}, {'word': ' one,', 'start': 21.88, 'end': 21.88, 'probability': 0.9249057173728943}, {'word': ' one,', 'start': 22.76, 'end': 22.8, 'probability': 0.6350448131561279}]}\n",
">>>> {'id': 4, 'seek': 0, 'start': 22.9, 'end': 22.9, 'text': '', 'tokens': [], 'temperature': 0.0, 'avg_logprob': -0.3545205489448879, 'compression_ratio': 1.8932038834951457, 'no_speech_prob': 0.48473232984542847, 'words': []}\n",
">>>> {'id': 5, 'seek': 0, 'start': 22.94, 'end': 23.68, 'text': ' One, two, one, one, one, one', 'tokens': [51313, 1881, 11, 734, 11, 530, 11, 530, 11, 530, 11, 530, 51513], 'temperature': 0.0, 'avg_logprob': -0.3545205489448879, 'compression_ratio': 1.8932038834951457, 'no_speech_prob': 0.48473232984542847, 'words': [{'word': ' one', 'start': 22.94, 'end': 22.94, 'probability': 0.4457760453224182}, {'word': ' Been', 'start': 22.94, 'end': 23.38, 'probability': 0.017312727868556976}, {'word': ' me', 'start': 23.38, 'end': 23.68, 'probability': 0.674941897392273}]}\n",
">>>> {'id': 6, 'seek': 0, 'start': 23.0, 'end': 25.0, 'text': ' Been me', 'tokens': [51513, 36844, 502, 51613], 'temperature': 0.0, 'avg_logprob': -0.3545205489448879, 'compression_ratio': 1.8932038834951457, 'no_speech_prob': 0.48473232984542847, 'words': []}\n",
">>>> {'id': 7, 'seek': 2368, 'start': 24.28, 'end': 28.22, 'text': ' One, two, one, one, one, one', 'tokens': [50413, 1881, 11, 734, 11, 530, 11, 530, 11, 530, 11, 530, 50563], 'temperature': 0.0, 'avg_logprob': -0.21509367889828152, 'compression_ratio': 1.2, 'no_speech_prob': 0.6877052783966064, 'words': [{'word': ' One,', 'start': 24.28, 'end': 24.84, 'probability': 0.006241263821721077}, {'word': ' two,', 'start': 25.08, 'end': 25.56, 'probability': 0.32598963379859924}, {'word': ' one,', 'start': 26.1, 'end': 26.4, 'probability': 0.7222583889961243}, {'word': ' one,', 'start': 26.6, 'end': 26.86, 'probability': 0.9019818305969238}, {'word': ' one,', 'start': 27.0, 'end': 27.4, 'probability': 0.8474482297897339}, {'word': ' one', 'start': 27.54, 'end': 27.54, 'probability': 0.5728994607925415}, {'word': ' Been', 'start': 27.54, 'end': 27.96, 'probability': 4.6182540245354176e-05}, {'word': ' me', 'start': 27.96, 'end': 28.22, 'probability': 0.008603227324783802}]}\n",
">>>> {'id': 8, 'seek': 2368, 'start': 27.68, 'end': 28.68, 'text': ' Been me', 'tokens': [50563, 36844, 502, 50613], 'temperature': 0.0, 'avg_logprob': -0.21509367889828152, 'compression_ratio': 1.2, 'no_speech_prob': 0.6877052783966064, 'words': []}\n"
]
}
],
"source": [
"for s in history_text['segments']:\n",
" print(\">>>> \", s)\n",
"# print(history_text['segments'])\n",
"# print(transcription)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "000f3100-f770-420a-80fe-9428e5c86315",
"metadata": {},
"outputs": [],
"source": [
"def count_syllables_word(word):\n",
" if not word:\n",
" return 0\n",
" \n",
" word = word.lower()\n",
" vowels = \"aeiouy\"\n",
" count = 0\n",
"\n",
" if word[0] in vowels:\n",
" count += 1\n",
"\n",
" for i in range(1, len(word)):\n",
" if word[i] in vowels and word[i - 1] not in vowels:\n",
" count += 1\n",
"\n",
" if word.endswith('e'):\n",
" count -= 1\n",
"\n",
" if word.endswith('le'):\n",
" count += 1\n",
"\n",
" if count == 0:\n",
" count += 1\n",
"\n",
" return count\n",
"\n",
"def count_syllables(text):\n",
" word_arr = text.split()\n",
" count = 0\n",
" for word in word_arr:\n",
" count += count_syllables_word(word)\n",
" return count"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "e8e11d89-5862-4041-8ef0-b9ee079379ea",
"metadata": {},
"outputs": [],
"source": [
"# syllables_w = 0\n",
"# syllables_r = 0\n",
"# for i in range(len(words)):\n",
"# w = words[i]\n",
"# word_whis = w['word']\n",
"# word_real = real[i] if i < len(real) else \"\"\n",
"# syllables_w += count_syllables(word_whis)\n",
"# syllables_r += count_syllables(word_real)\n",
"# print (word_whis, ' ', w['end'], ' ', syllables_w, ' | ', word_real, ' ',syllables_r)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "ad1eb4a5-6a71-4c3c-b6d3-a4eadc7c0cb3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'word': 'I told the witch doctor, I was in love with you', 'syllables': 12}\n",
"MATCH: 10 52 {'word': ' you', 'syllables': 12, 'start': 4.38, 'end': 4.58}\n",
"{'word': 'I told the witch doctor, I was in love with you', 'syllables': 24}\n",
"MATCH: 21 52 {'word': ' you', 'syllables': 24, 'start': 8.94, 'end': 9.26}\n",
"{'word': 'And then the witch doctor, he told me what to do', 'syllables': 36}\n",
"MATCH: 32 52 {'word': ' do', 'syllables': 36, 'start': 13.5, 'end': 14.04}\n",
"{'word': 'He told me:', 'syllables': 39}\n",
"MATCH: 35 52 {'word': ' me', 'syllables': 39, 'start': 15.0, 'end': 15.32}\n",
"{'word': '', 'syllables': 39}\n",
"MATCH: 35 52 {'word': ' me', 'syllables': 39, 'start': 15.0, 'end': 15.32}\n",
"{'word': 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 'syllables': 52}\n",
"MATCH: 44 52 {'word': ' One,', 'syllables': 52, 'start': 24.28, 'end': 24.84}\n",
"{'word': 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 'syllables': 65}\n",
"{'word': '', 'syllables': 65}\n"
]
}
],
"source": [
"# 1. Store syllables up to each word in the original and transcribed text\n",
"# print (len(prompt))\n",
"real_text = [] \n",
"for i in range(len(prompt)):\n",
" word = prompt[i]\n",
" val = {}\n",
" val['word'] = word\n",
" val['syllables'] = count_syllables(word) + (real_text[i-1]['syllables'] if i > 0 else 0)\n",
" real_text.append(val)\n",
" # print (real_text[i])\n",
"\n",
"transcribed_text = []\n",
"for i in range(len(transcription)):\n",
" word = transcription[i]['word']\n",
" val = {}\n",
" val['word'] = word\n",
" val['syllables'] = count_syllables(word) + (transcribed_text[i-1]['syllables'] if i > 0 else 0)\n",
" val['start'] = transcription[i]['start']\n",
" val['end'] = transcription[i]['end']\n",
" transcribed_text.append(val)\n",
" # print(i, transcribed_text[i])\n",
" \n",
"# 2. Iterate through /n in original text. \n",
" # Find first word in the transcribed text that has >= syllables and use that as start time for new line\n",
"transcribe_word_count = 0\n",
"for t in real_text:\n",
" print(t)\n",
" while transcribe_word_count < len(transcribed_text):\n",
" transcribed_word = transcribed_text[transcribe_word_count]\n",
" # if transcribe_word_count > 74:\n",
" # print(\" \", transcribe_word_count, transcribed_word)\n",
" if transcribed_word['syllables'] >= t['syllables']:\n",
" t['end_time'] = transcribed_word['end']\n",
" print (\"MATCH: \", transcribe_word_count, len(transcribed_text), transcribed_word)\n",
" break\n",
" transcribe_word_count+=1\n",
"\n",
"# print(real_text) \n",
"#real_text contains each line. end_time is the start time of the following line\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "ae6f2173-6338-4e12-9159-32233c91fd63",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.worker import video_seewav\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "b10eb56c-5f34-46b4-b65b-6c4ef7257109",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 {'word': 'I told the witch doctor, I was in love with you', 'syllables': 12, 'end_time': 4.58}\n",
"1 {'word': 'I told the witch doctor, I was in love with you', 'syllables': 24, 'end_time': 9.26}\n",
"2 {'word': 'And then the witch doctor, he told me what to do', 'syllables': 36, 'end_time': 14.04}\n",
"3 {'word': 'He told me:', 'syllables': 39, 'end_time': 15.32}\n",
"4 {'word': '', 'syllables': 39, 'end_time': 15.32}\n",
"5 {'word': 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 'syllables': 52, 'end_time': 24.84}\n",
"6 {'word': 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 'syllables': 65}\n",
"7 {'word': '', 'syllables': 65}\n",
"[{'l': 'I told the witch doctor, I was in love with you', 't': 0}, {'l': 'I told the witch doctor, I was in love with you', 't': 5}, {'l': 'And then the witch doctor, he told me what to do', 't': 9}, {'l': 'He told me:', 't': 14}, {'l': 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 't': 15}, {'l': 'Ooh, eeh, ooh, ah, ah, ting, tang, walla, walla, bing, bang', 't': 25}]\n"
]
}
],
"source": [
"timestamps = []\n",
"for i in range(len(real_text)):\n",
" val = {}\n",
" val['l'] = real_text[i]['word']\n",
" print(i, real_text[i])\n",
" end_time_str = (real_text[i-1].get('end_time') if i > 0 else 0)\n",
" val['t'] = round(float(end_time_str)) if end_time_str is not None else timestamps[- 1]['t']\n",
" if(i > 0 and val['t'] == timestamps[- 1]['t']):\n",
" timestamps[- 1]['l'] += val['l']\n",
" # elif(i >= len(timestamps)):\n",
" # print(len(timestamps))\n",
" # timestamps[-1]['l'] += val['l']\n",
" else:\n",
" timestamps.append(val)\n",
"\n",
"print(timestamps)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "f81e0e3e-13ea-4a46-9941-b199a95bca7d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[{'l': 'I told the witch doctor, I was in love with you\\nI told the witch doctor, I was in love with you\\nAnd then the witch doctor, he told me what to do\\nHe told me:', 't': 0}]\n"
]
}
],
"source": [
"timestamps_chunked = []\n",
"val = {}\n",
"num_chunks = 1\n",
"val['l'] = real_text[0]['word'] + \"\\n\" + real_text[1]['word']+ \"\\n\" + real_text[2]['word']+ \"\\n\" +real_text[3]['word']\n",
"val['t'] = 0\n",
"timestamps_chunked.append(val)\n",
"\n",
"for i in range(len(timestamps)):\n",
" val = {}\n",
" if i % 2 == 0 and i > 4:\n",
" #each chunk has two previous and two after\n",
" val['l'] = real_text[i-3]['word']+ \"\\n\" +real_text[i-2]['word']+ \"\\n\" +real_text[i-1]['word']+ \"\\n\" +real_text[i]['word']\n",
" val['t'] = round(float(real_text[i-3].get('end_time') if i > 0 else 0))\n",
" timestamps.append(val)\n",
"\n",
"print(timestamps_chunked)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "9411119e-e764-480e-bc87-c7e7d703cad5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"393 -393\n",
"Generating the frames...\n",
"Generated 150 frames in 4.79 seconds.\n",
"393 -393\n",
"Generating the frames...\n",
"Generated 120 frames in 4.11 seconds.\n",
"395 -395\n",
"Generating the frames...\n",
"Generated 150 frames in 4.64 seconds.\n",
"165 -165\n",
"Generating the frames...\n",
"Generated 30 frames in 2.48 seconds.\n",
"390 -390\n",
"Generating the frames...\n",
"Generated 300 frames in 7.28 seconds.\n",
"390 -390\n",
"Generating the frames...\n",
"Generated 150 frames in 4.56 seconds.\n",
"Encoding the animation video... \n",
"Encoded the video in 1.72 seconds.\n"
]
},
{
"data": {
"text/plain": [
"PosixPath('out.mp4')"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"video_seewav.render_demucs_audio(\n",
" a, prompt_raw,\n",
" out=\"out.mp4\", png_path=png_path, is_square=True, image_transparency=0.4, timestamps=timestamps\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2d4dde73-4b5f-4a93-b8ef-92fb62db75e8",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d406b9a6-8fd3-4f1f-8f84-a25b35df1fb2",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}