{
"cells": [
{
"cell_type": "markdown",
"id": "f8ad1195",
"metadata": {},
"source": [
"### Test general stuff"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "7a481316",
"metadata": {},
"outputs": [],
"source": [
"# https://github.com/marin-m/SongRec\n",
"# https://github.com/Numenorean/ShazamAPI\n",
"# https://docs.genius.com/"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "11056054",
"metadata": {},
"outputs": [],
"source": [
"import lyricsgenius as lg\n",
"import json\n",
"\n",
"with open(\"/home/georg/.secrets/secrets.json\") as f:\n",
" secrets = json.load(f)\n",
" \n",
"GENIUS_ACCESS_TOKEN = secrets[\"genius_access_token\"]"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "b3048611",
"metadata": {},
"outputs": [],
"source": [
"# https://genius.com/Nicki-minaj-and-lil-baby-do-we-have-a-problem-lyrics"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "14ca2c70",
"metadata": {},
"outputs": [],
"source": [
"genius = lg.Genius(\n",
" GENIUS_ACCESS_TOKEN, \n",
" skip_non_songs=True, \n",
" excluded_terms=[\"(Remix)\", \"(Live)\"], \n",
" remove_section_headers=True,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "3816b53f",
"metadata": {},
"outputs": [],
"source": [
"lyrics_guess = \"look at my trainers are froze i ain't come over here for nothing no he is\"\n",
"out = genius.search_lyrics(lyrics_guess, per_page=10, page=1)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "7296a495",
"metadata": {},
"outputs": [],
"source": [
"song_id = out[\"sections\"][0][\"hits\"][0][\"result\"][\"id\"]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "b0ea980b",
"metadata": {},
"outputs": [],
"source": [
"out_lyrics = genius.lyrics(song_id=song_id, remove_section_headers=True)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "fbf550a2",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Big jimmy 2018 MY FIRST DRAFT MIXTAPE LyricsTen songs\n",
"The love of my family ✅\n",
"I’m a Modern day Arist\n"
]
}
],
"source": [
"print(out_lyrics[:100])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "52faba99",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "0c1ede2c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "c8742307",
"metadata": {},
"source": [
"### Try in bulk"
]
},
{
"cell_type": "code",
"execution_count": 78,
"id": "36f12402",
"metadata": {},
"outputs": [],
"source": [
"import lyricsgenius as lg\n",
"import json\n",
"from suno_utils.audio.conversion import play_audio\n",
"\n",
"with open(\"/home/georg/.secrets/secrets.json\") as f:\n",
" secrets = json.load(f)\n",
" \n",
"GENIUS_ACCESS_TOKEN = secrets[\"genius_access_token\"]\n",
"\n",
"genius = lg.Genius(\n",
" GENIUS_ACCESS_TOKEN, \n",
" skip_non_songs=True, \n",
" excluded_terms=[\"(Remix)\", \"(Live)\"], \n",
" remove_section_headers=True,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "cc1fb4cb",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"BASE_DIR = \"/mnt/data-ssd-1/data/private/customer/speechly/2022-08-19_tiktok_250\"\n",
"SEGMENT_DATE_DIR = \"2022_08_19\"\n",
"ARTIFACTS_DIR = os.path.join(BASE_DIR, \"pipeline\", SEGMENT_DATE_DIR, \"artifacts\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "0fbbcfda",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"from suno_utils.audio import Tokens\n",
"\n",
"with open(os.path.join(ARTIFACTS_DIR, \"01_segment_meta.json\")) as f:\n",
" segments_manifest = json.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "40ff27e3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"58 found\n"
]
}
],
"source": [
"import numpy as np\n",
"music_idx_list = []\n",
"for n in range(250):\n",
" tokens = Tokens.from_dict(segments_manifest[n][\"transcript\"][\"tokens\"])\n",
" if all([t.metadata[\"is_lyrics\"] for t in tokens if t.type == \"text\"]):\n",
" music_idx_list.append(n)\n",
"print(len(music_idx_list), \"found\")"
]
},
{
"cell_type": "code",
"execution_count": 76,
"id": "38b93b09",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[music] That ain't shit to me. Same dude different drink this is me this is me Ain't nothing pretty but her face. Money hungry, nothing skinny but her waist.\n"
]
}
],
"source": [
"n = 0\n",
"tokens = Tokens.from_dict(segments_manifest[music_idx_list[0]][\"transcript\"][\"tokens\"])\n",
"print(tokens.text)"
]
},
{
"cell_type": "code",
"execution_count": 87,
"id": "d4e7e8c8",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"play_audio(segments_manifest[music_idx_list[0]][\"uri\"])"
]
},
{
"cell_type": "code",
"execution_count": 75,
"id": "58a2868e",
"metadata": {},
"outputs": [],
"source": [
"# for n in range(250):\n",
"# tokens = Tokens.from_dict(segments_manifest[n][\"transcript\"][\"tokens\"])\n",
"# text = tokens.plaintext\n",
"# if \"gettin' shot\" in text.lower():\n",
"# print(n)\n",
"# print(text)\n",
"# break"
]
},
{
"cell_type": "code",
"execution_count": 79,
"id": "35dc9f72",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mike. fka mike stud\n",
"These Days\n"
]
}
],
"source": [
"lyrics_guess = tokens.plaintext\n",
"out = genius.search_lyrics(lyrics_guess, per_page=10, page=1)\n",
"best_match = out[\"sections\"][0][\"hits\"][0][\"result\"]\n",
"# TODO: any way to check the match quality?\n",
"print(best_match[\"artist_names\"])\n",
"print(best_match[\"title\"])\n",
"print(best_match[\"url\"])\n",
"song_id = best_match[\"id\"]\n",
"out_lyrics = genius.lyrics(song_id=song_id, remove_section_headers=True)\n",
"# TODO: make sure are all headers gone"
]
},
{
"cell_type": "code",
"execution_count": 90,
"id": "a4abb409",
"metadata": {},
"outputs": [],
"source": [
"# print(out_lyrics)"
]
},
{
"cell_type": "code",
"execution_count": 106,
"id": "38ad0d3a",
"metadata": {},
"outputs": [],
"source": [
"import re\n",
"from Bio import pairwise2\n",
"from suno_utils.utils.text import normalize_whitespace\n",
"\n",
"def _simplify(s):\n",
" s = s.lower()\n",
" s = re.sub(r\"[^a-z0-9\\']\", \"\", s)\n",
" return s\n",
"\n",
"full_lyrics = normalize_whitespace(out_lyrics)\n",
"needle_list = tokens.plaintext.split()\n",
"haystack_list = full_lyrics.split()\n",
"# make normed version and keep maps\n",
"norm_needle_list = [ss for s in needle_list if len(ss := _simplify(s)) > 0]\n",
"haystack_idx_map = {}\n",
"norm_haystack_list = []\n",
"for n, s in enumerate(haystack_list):\n",
" ss = _simplify(s)\n",
" if len(ss) == 0:\n",
" continue\n",
" haystack_idx_map[len(norm_haystack_list)] = n\n",
" norm_haystack_list.append(ss)\n",
"# do alignment\n",
"out = pairwise2.align.globalms(\n",
" norm_needle_list, \n",
" norm_haystack_list, \n",
" gap_char=[\"*\"],\n",
" match=2, mismatch=-1, open=-.5, extend=-.1,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 145,
"id": "6e1f3105",
"metadata": {},
"outputs": [],
"source": [
"out = pairwise2.align.globalms(\n",
" norm_needle_list, \n",
" norm_haystack_list, \n",
" gap_char=[\"*\"],\n",
" penalize_extend_when_opening=False,\n",
" penalize_end_gaps=False,\n",
" match=2, mismatch=-1, open=-.5, extend=-.1\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 146,
"id": "2e8fb182",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'lyrics'),\n",
" ('*', 'hell'),\n",
" ('*', 'nah'),\n",
" ('*', 'i'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'sleep'),\n",
" ('*', 'today'),\n",
" ('*', 'but'),\n",
" ('*', \"i'ma\"),\n",
" ('*', 'be'),\n",
" ('*', 'okay'),\n",
" ('*', 'miss'),\n",
" ('*', 'my'),\n",
" ('*', 'family'),\n",
" ('*', 'on'),\n",
" ('*', 'the'),\n",
" ('*', 'east'),\n",
" ('*', 'but'),\n",
" ('*', 'fuck'),\n",
" ('*', 'it'),\n",
" ('*', 'i'),\n",
" ('*', \"can't\"),\n",
" ('*', 'leave'),\n",
" ('*', 'la'),\n",
" ('*', 'every'),\n",
" ('*', 'meeting'),\n",
" ('*', 'that'),\n",
" ('*', 'i'),\n",
" ('*', 'be'),\n",
" ('*', 'in'),\n",
" ('*', 'trust'),\n",
" ('*', 'me'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'lead'),\n",
" ('*', 'the'),\n",
" ('*', 'way'),\n",
" ('*', 'i'),\n",
" ('*', 'fucked'),\n",
" ('*', 'three'),\n",
" ('*', 'times'),\n",
" ('*', 'before'),\n",
" ('*', '3'),\n",
" ('*', 'today'),\n",
" ('*', 'piece'),\n",
" ('*', 'of'),\n",
" ('*', 'cake'),\n",
" ('*', 'never'),\n",
" ('*', 'gave'),\n",
" ('*', 'a'),\n",
" ('*', 'fuck'),\n",
" ('*', 'about'),\n",
" ('*', 'what'),\n",
" ('*', 'the'),\n",
" ('*', 'teacher'),\n",
" ('*', 'say'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'either'),\n",
" ('*', 'way'),\n",
" ('*', \"i'mma\"),\n",
" ('*', 'just'),\n",
" ('*', 'do'),\n",
" ('*', 'me'),\n",
" ('*', 'and'),\n",
" ('*', 'she'),\n",
" ('*', 'said'),\n",
" ('*', 'me'),\n",
" ('*', 'too'),\n",
" ('*', 'i'),\n",
" ('*', 'said'),\n",
" ('*', \"s'il\"),\n",
" ('*', 'vous'),\n",
" ('*', 'plat'),\n",
" ('*', 'walking'),\n",
" ('*', 'out'),\n",
" ('*', 'the'),\n",
" ('*', 'club'),\n",
" ('*', 'with'),\n",
" ('*', 'me'),\n",
" ('*', 'and'),\n",
" ('*', 'tmz'),\n",
" ('*', 'is'),\n",
" ('*', 'out'),\n",
" ('*', 'here'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'no'),\n",
" ('*', 'secret'),\n",
" ('*', 'safe'),\n",
" ('*', 'if'),\n",
" ('*', \"that's\"),\n",
" ('*', 'your'),\n",
" ('*', 'girl'),\n",
" ('*', \"why's\"),\n",
" ('*', 'she'),\n",
" ('*', 'in'),\n",
" ('*', 'my'),\n",
" ('*', 'section'),\n",
" ('*', 'if'),\n",
" ('*', \"that's\"),\n",
" ('*', 'your'),\n",
" ('*', 'girl'),\n",
" ('*', \"why's\"),\n",
" ('*', 'she'),\n",
" ('*', 'doing'),\n",
" ('*', 'role'),\n",
" ('*', 'play'),\n",
" ('*', 'going'),\n",
" ('*', 'both'),\n",
" ('*', 'ways'),\n",
" ('*', 'like'),\n",
" ('*', 'an'),\n",
" ('*', 'intersection'),\n",
" ('*', 'stole'),\n",
" ('*', 'her'),\n",
" ('*', 'like'),\n",
" ('*', 'an'),\n",
" ('*', 'interception'),\n",
" ('*', 'but'),\n",
" ('*', 'you'),\n",
" ('*', 'can'),\n",
" ('*', 'have'),\n",
" ('*', 'her'),\n",
" ('*', 'back'),\n",
" ('*', 'she'),\n",
" ('*', 'asks'),\n",
" ('*', 'a'),\n",
" ('*', 'million'),\n",
" ('*', 'questions'),\n",
" ('*', 'like'),\n",
" ('*', \"what's\"),\n",
" ('*', 'next'),\n",
" ('*', \"what's\"),\n",
" ('*', 'that'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'wanna'),\n",
" ('*', 'chill'),\n",
" ('*', 'drink'),\n",
" ('*', 'smoke'),\n",
" ('*', 'fuck'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'god'),\n",
" ('*', 'damn'),\n",
" ('*', 'that'),\n",
" ('*', \"body's\"),\n",
" ('*', 'goals'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'somehow'),\n",
" ('*', 'you'),\n",
" ('*', 'hot'),\n",
" ('*', 'as'),\n",
" ('*', 'hell'),\n",
" ('*', 'but'),\n",
" ('*', 'still'),\n",
" ('*', 'cold'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', 'the'),\n",
" ('*', 'deal'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'you'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'the'),\n",
" ('*', 'only'),\n",
" ('*', 'one'),\n",
" ('*', 'you'),\n",
" ('*', 'knew'),\n",
" ('*', 'that'),\n",
" ('*', 'everything'),\n",
" ('*', 'i'),\n",
" ('*', 'see'),\n",
" ('*', 'up'),\n",
" ('*', 'in'),\n",
" ('*', 'my'),\n",
" ('*', 'head'),\n",
" ('*', 'just'),\n",
" ('*', 'comes'),\n",
" ('*', 'to'),\n",
" ('*', 'life'),\n",
" ('*', 'got'),\n",
" ('*', 'my'),\n",
" ('*', 'favorite'),\n",
" ('*', 'girl'),\n",
" ('*', 'waking'),\n",
" ('*', 'up'),\n",
" ('*', 'in'),\n",
" ('*', 'my'),\n",
" ('*', 'bed'),\n",
" ('*', \"it's\"),\n",
" ('*', 'only'),\n",
" ('*', 'right'),\n",
" ('*', 'bad'),\n",
" ('*', 'bitch'),\n",
" ('*', 'my'),\n",
" ('*', 'only'),\n",
" ('*', 'type'),\n",
" ('*', 'independent'),\n",
" ('*', 'too'),\n",
" ('*', 'get'),\n",
" ('*', 'it'),\n",
" ('*', 'boo'),\n",
" ('*', 'just'),\n",
" ('*', 'promise'),\n",
" ('*', 'me'),\n",
" ('*', 'you'),\n",
" ('*', \"won't\"),\n",
" ('*', 'let'),\n",
" ('*', 'all'),\n",
" ('*', 'the'),\n",
" ('*', 'bullshit'),\n",
" ('*', 'they'),\n",
" ('*', 'say'),\n",
" ('*', 'get'),\n",
" ('*', 'to'),\n",
" ('*', 'you'),\n",
" ('*', 'you'),\n",
" ('*', 'a'),\n",
" ('*', '10'),\n",
" ('*', \"i'm\"),\n",
" ('*', 'not'),\n",
" ('*', 'perfect'),\n",
" ('*', 'but'),\n",
" ('*', 'for'),\n",
" ('*', 'you'),\n",
" ('*', 'i'),\n",
" ('*', 'do'),\n",
" ('*', 'pretend'),\n",
" ('*', \"i've\"),\n",
" ('*', 'been'),\n",
" ('*', 'working'),\n",
" ('*', '2'),\n",
" ('*', 'to'),\n",
" ('*', '10'),\n",
" ('*', \"that's\"),\n",
" ('*', 'am'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'nobody'),\n",
" ('*', 'quite'),\n",
" ('*', 'like'),\n",
" ('*', 'him'),\n",
" ('*', 'put'),\n",
" ('*', 'in'),\n",
" ('*', 'the'),\n",
" ('*', 'time'),\n",
" ('*', 'like'),\n",
" ('*', 'him'),\n",
" ('*', \"i've\"),\n",
" ('*', 'been'),\n",
" ('*', 'grinding'),\n",
" ('*', 'for'),\n",
" ('*', 'a'),\n",
" ('*', 'while'),\n",
" ('*', 'now'),\n",
" ('*', \"it's\"),\n",
" ('*', 'feeling'),\n",
" ('*', 'like'),\n",
" ('*', 'my'),\n",
" ('*', 'time'),\n",
" ('*', 'is'),\n",
" ('*', 'finally'),\n",
" ('*', 'here'),\n",
" ('*', \"that's\"),\n",
" ('*', 'ideal'),\n",
" ('*', \"that's\"),\n",
" ('*', 'ideal'),\n",
" ('*', 'like'),\n",
" ('*', \"what's\"),\n",
" ('*', 'next'),\n",
" ('*', \"what's\"),\n",
" ('*', 'that'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'wanna'),\n",
" ('*', 'chill'),\n",
" ('*', 'drink'),\n",
" ('*', 'smoke'),\n",
" ('*', 'fuck'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'god'),\n",
" ('*', 'damn'),\n",
" ('*', 'that'),\n",
" ('*', \"body's\"),\n",
" ('*', 'goals'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'somehow'),\n",
" ('*', 'you'),\n",
" ('*', 'hot'),\n",
" ('*', 'as'),\n",
" ('*', 'hell'),\n",
" ('*', 'but'),\n",
" ('*', 'still'),\n",
" ('*', 'cold'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', 'the'),\n",
" ('*', 'deal'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'you'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'the'),\n",
" ('*', 'only'),\n",
" ('*', 'one'),\n",
" ('*', 'you'),\n",
" ('*', 'knew'),\n",
" ('*', 'that'),\n",
" ('*', \"i've\"),\n",
" ('*', 'been'),\n",
" ('*', 'doing'),\n",
" ('*', 'great'),\n",
" ('*', 'i'),\n",
" ('*', 'guess'),\n",
" ('*', 'i'),\n",
" ('*', \"can't\"),\n",
" ('*', 'complain'),\n",
" ('*', 'i'),\n",
" ('*', \"don't\"),\n",
" ('*', 'think'),\n",
" ('*', 'about'),\n",
" ('*', 'yesterday'),\n",
" ('*', 'every'),\n",
" ('*', 'move'),\n",
" ('*', 'is'),\n",
" ('*', 'calculated'),\n",
" ('*', 'this'),\n",
" ('*', 'shit'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'no'),\n",
" ('*', 'guessing'),\n",
" ('*', 'game'),\n",
" ('*', 'only'),\n",
" ('*', 'time'),\n",
" ('*', \"i'm\"),\n",
" ('*', 'second'),\n",
" ('*', 'place'),\n",
" ('*', 'is'),\n",
" ('*', 'when'),\n",
" ('*', \"i'm\"),\n",
" ('*', 'at'),\n",
" ('*', 'my'),\n",
" ('*', 'second'),\n",
" ('*', 'place'),\n",
" ('*', 'made'),\n",
" ('*', '300'),\n",
" ('*', 'then'),\n",
" ('*', 'doubled'),\n",
" ('*', 'that'),\n",
" ('*', 'i'),\n",
" ('*', 'guess'),\n",
" ('*', 'you'),\n",
" ('*', 'call'),\n",
" ('*', 'that'),\n",
" ('*', '2nd'),\n",
" ('*', 'base'),\n",
" ('*', 'how'),\n",
" ('*', 'many'),\n",
" ('*', 'come'),\n",
" ('*', 'ups'),\n",
" ('*', 'until'),\n",
" ('*', \"it's\"),\n",
" ('*', 'destiny'),\n",
" ('*', 'how'),\n",
" ('*', 'many'),\n",
" ('*', 'come'),\n",
" ('*', 'ups'),\n",
" ('*', 'until'),\n",
" ('*', \"it's\"),\n",
" ('*', 'meant'),\n",
" ('*', 'to'),\n",
" ('*', 'be'),\n",
" ('*', 'i'),\n",
" ('*', 'hear'),\n",
" ('*', 'them'),\n",
" ('*', 'talking'),\n",
" ('*', 'that'),\n",
" ('*', \"don't\"),\n",
" ('*', 'get'),\n",
" ('*', 'to'),\n",
" ('*', 'me'),\n",
" ('that', 'that'),\n",
" (\"ain't\", \"ain't\"),\n",
" ('shit', 'shit'),\n",
" ('to', 'to'),\n",
" ('me', 'me'),\n",
" ('same', 'same'),\n",
" ('dude', 'dude'),\n",
" ('different', 'different'),\n",
" ('drink', '*'),\n",
" ('this', '*'),\n",
" ('is', '*'),\n",
" ('me', '*'),\n",
" ('*', 'dream'),\n",
" ('this', 'this'),\n",
" ('is', 'is'),\n",
" ('me', 'me'),\n",
" (\"ain't\", \"ain't\"),\n",
" ('nothing', 'nothing'),\n",
" ('pretty', 'pretty'),\n",
" ('but', 'but'),\n",
" ('her', 'her'),\n",
" ('face', 'face'),\n",
" ('money', 'money'),\n",
" ('hungry', 'hungry'),\n",
" ('nothing', 'nothing'),\n",
" ('skinny', 'skinny'),\n",
" ('but', 'but'),\n",
" ('her', 'her'),\n",
" ('waist', 'waist'),\n",
" ('*', 'she'),\n",
" ('*', 'come'),\n",
" ('*', 'from'),\n",
" ('*', 'money'),\n",
" ('*', 'but'),\n",
" ('*', 'she'),\n",
" ('*', 'spent'),\n",
" ('*', 'it'),\n",
" ('*', 'on'),\n",
" ('*', 'her'),\n",
" ('*', 'titties'),\n",
" ('*', 'and'),\n",
" ('*', 'her'),\n",
" ('*', 'face'),\n",
" ('*', 'if'),\n",
" ('*', 'you'),\n",
" ('*', 'really'),\n",
" ('*', 'wanna'),\n",
" ('*', 'know'),\n",
" ('*', 'these'),\n",
" ('*', 'women'),\n",
" ('*', 'man'),\n",
" ('*', 'they'),\n",
" ('*', 'come'),\n",
" ('*', 'and'),\n",
" ('*', 'go'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'one'),\n",
" ('*', 'minute'),\n",
" ('*', \"she's\"),\n",
" ('*', 'the'),\n",
" ('*', 'one'),\n",
" ('*', 'the'),\n",
" ('*', 'next'),\n",
" ('*', 'you'),\n",
" ('*', 'really'),\n",
" ('*', 'never'),\n",
" ('*', 'know'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'she'),\n",
" ('*', \"don't\"),\n",
" ('*', 'know'),\n",
" ('*', 'a'),\n",
" ('*', 'damn'),\n",
" ('*', 'thing'),\n",
" ('*', 'about'),\n",
" ('*', 'love'),\n",
" ('*', 'she'),\n",
" ('*', \"don't\"),\n",
" ('*', 'know'),\n",
" ('*', 'a'),\n",
" ('*', 'damn'),\n",
" ('*', 'thing'),\n",
" ('*', 'at'),\n",
" ('*', 'all'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'one'),\n",
" ('*', 'minute'),\n",
" ('*', \"she's\"),\n",
" ('*', 'the'),\n",
" ('*', 'one'),\n",
" ('*', 'the'),\n",
" ('*', 'next'),\n",
" ('*', 'you'),\n",
" ('*', 'really'),\n",
" ('*', 'never'),\n",
" ('*', 'know'),\n",
" ('*', 'like'),\n",
" ('*', \"what's\"),\n",
" ('*', 'next'),\n",
" ('*', \"what's\"),\n",
" ('*', 'that'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'wanna'),\n",
" ('*', 'chill'),\n",
" ('*', 'drink'),\n",
" ('*', 'smoke'),\n",
" ('*', 'fuck'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'god'),\n",
" ('*', 'damn'),\n",
" ('*', 'that'),\n",
" ('*', \"body's\"),\n",
" ('*', 'goals'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'somehow'),\n",
" ('*', 'you'),\n",
" ('*', 'hot'),\n",
" ('*', 'as'),\n",
" ('*', 'hell'),\n",
" ('*', 'but'),\n",
" ('*', 'still'),\n",
" ('*', 'cold'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', 'the'),\n",
" ('*', 'deal'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'you'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'the'),\n",
" ('*', 'only'),\n",
" ('*', 'one'),\n",
" ('*', 'girls'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'you'),\n",
" ('*', 'ready'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'uh'),\n",
" ('*', 'you'),\n",
" ('*', 'ready'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'you'),\n",
" ('*', 'ready'),\n",
" ('*', 'girls'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'girls'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'these'),\n",
" ('*', 'days42embed')]"
]
},
"execution_count": 146,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(zip(out[0].seqA, out[0].seqB))"
]
},
{
"cell_type": "code",
"execution_count": 108,
"id": "d6217a95",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'lyrics'),\n",
" ('*', 'hell'),\n",
" ('*', 'nah'),\n",
" ('*', 'i'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'sleep'),\n",
" ('*', 'today'),\n",
" ('*', 'but'),\n",
" ('*', \"i'ma\"),\n",
" ('*', 'be'),\n",
" ('*', 'okay'),\n",
" ('*', 'miss'),\n",
" ('*', 'my'),\n",
" ('*', 'family'),\n",
" ('*', 'on'),\n",
" ('*', 'the'),\n",
" ('*', 'east'),\n",
" ('*', 'but'),\n",
" ('*', 'fuck'),\n",
" ('*', 'it'),\n",
" ('*', 'i'),\n",
" ('*', \"can't\"),\n",
" ('*', 'leave'),\n",
" ('*', 'la'),\n",
" ('*', 'every'),\n",
" ('*', 'meeting'),\n",
" ('that', 'that'),\n",
" ('*', 'i'),\n",
" ('*', 'be'),\n",
" ('*', 'in'),\n",
" ('*', 'trust'),\n",
" ('*', 'me'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'lead'),\n",
" ('*', 'the'),\n",
" ('*', 'way'),\n",
" ('*', 'i'),\n",
" ('*', 'fucked'),\n",
" ('*', 'three'),\n",
" ('*', 'times'),\n",
" ('*', 'before'),\n",
" ('*', '3'),\n",
" ('*', 'today'),\n",
" ('*', 'piece'),\n",
" ('*', 'of'),\n",
" ('*', 'cake'),\n",
" ('*', 'never'),\n",
" ('*', 'gave'),\n",
" ('*', 'a'),\n",
" ('*', 'fuck'),\n",
" ('*', 'about'),\n",
" ('*', 'what'),\n",
" ('*', 'the'),\n",
" ('*', 'teacher'),\n",
" ('*', 'say'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'either'),\n",
" ('*', 'way'),\n",
" ('*', \"i'mma\"),\n",
" ('*', 'just'),\n",
" ('*', 'do'),\n",
" ('*', 'me'),\n",
" ('*', 'and'),\n",
" ('*', 'she'),\n",
" ('*', 'said'),\n",
" ('*', 'me'),\n",
" ('*', 'too'),\n",
" ('*', 'i'),\n",
" ('*', 'said'),\n",
" ('*', \"s'il\"),\n",
" ('*', 'vous'),\n",
" ('*', 'plat'),\n",
" ('*', 'walking'),\n",
" ('*', 'out'),\n",
" ('*', 'the'),\n",
" ('*', 'club'),\n",
" ('*', 'with'),\n",
" ('*', 'me'),\n",
" ('*', 'and'),\n",
" ('*', 'tmz'),\n",
" ('*', 'is'),\n",
" ('*', 'out'),\n",
" ('*', 'here'),\n",
" (\"ain't\", \"ain't\"),\n",
" ('shit', '*'),\n",
" ('*', 'no'),\n",
" ('*', 'secret'),\n",
" ('*', 'safe'),\n",
" ('*', 'if'),\n",
" ('*', \"that's\"),\n",
" ('*', 'your'),\n",
" ('*', 'girl'),\n",
" ('*', \"why's\"),\n",
" ('*', 'she'),\n",
" ('*', 'in'),\n",
" ('*', 'my'),\n",
" ('*', 'section'),\n",
" ('*', 'if'),\n",
" ('*', \"that's\"),\n",
" ('*', 'your'),\n",
" ('*', 'girl'),\n",
" ('*', \"why's\"),\n",
" ('*', 'she'),\n",
" ('*', 'doing'),\n",
" ('*', 'role'),\n",
" ('*', 'play'),\n",
" ('*', 'going'),\n",
" ('*', 'both'),\n",
" ('*', 'ways'),\n",
" ('*', 'like'),\n",
" ('*', 'an'),\n",
" ('*', 'intersection'),\n",
" ('*', 'stole'),\n",
" ('*', 'her'),\n",
" ('*', 'like'),\n",
" ('*', 'an'),\n",
" ('*', 'interception'),\n",
" ('*', 'but'),\n",
" ('*', 'you'),\n",
" ('*', 'can'),\n",
" ('*', 'have'),\n",
" ('*', 'her'),\n",
" ('*', 'back'),\n",
" ('*', 'she'),\n",
" ('*', 'asks'),\n",
" ('*', 'a'),\n",
" ('*', 'million'),\n",
" ('*', 'questions'),\n",
" ('*', 'like'),\n",
" ('*', \"what's\"),\n",
" ('*', 'next'),\n",
" ('*', \"what's\"),\n",
" ('*', 'that'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'wanna'),\n",
" ('*', 'chill'),\n",
" ('*', 'drink'),\n",
" ('*', 'smoke'),\n",
" ('*', 'fuck'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'god'),\n",
" ('*', 'damn'),\n",
" ('*', 'that'),\n",
" ('*', \"body's\"),\n",
" ('*', 'goals'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'somehow'),\n",
" ('*', 'you'),\n",
" ('*', 'hot'),\n",
" ('*', 'as'),\n",
" ('*', 'hell'),\n",
" ('*', 'but'),\n",
" ('*', 'still'),\n",
" ('*', 'cold'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', 'the'),\n",
" ('*', 'deal'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'you'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'the'),\n",
" ('*', 'only'),\n",
" ('*', 'one'),\n",
" ('*', 'you'),\n",
" ('*', 'knew'),\n",
" ('*', 'that'),\n",
" ('*', 'everything'),\n",
" ('*', 'i'),\n",
" ('*', 'see'),\n",
" ('*', 'up'),\n",
" ('*', 'in'),\n",
" ('*', 'my'),\n",
" ('*', 'head'),\n",
" ('*', 'just'),\n",
" ('*', 'comes'),\n",
" ('to', 'to'),\n",
" ('*', 'life'),\n",
" ('*', 'got'),\n",
" ('*', 'my'),\n",
" ('*', 'favorite'),\n",
" ('*', 'girl'),\n",
" ('*', 'waking'),\n",
" ('*', 'up'),\n",
" ('*', 'in'),\n",
" ('*', 'my'),\n",
" ('*', 'bed'),\n",
" ('*', \"it's\"),\n",
" ('*', 'only'),\n",
" ('*', 'right'),\n",
" ('*', 'bad'),\n",
" ('*', 'bitch'),\n",
" ('*', 'my'),\n",
" ('*', 'only'),\n",
" ('*', 'type'),\n",
" ('*', 'independent'),\n",
" ('*', 'too'),\n",
" ('*', 'get'),\n",
" ('*', 'it'),\n",
" ('*', 'boo'),\n",
" ('*', 'just'),\n",
" ('*', 'promise'),\n",
" ('me', 'me'),\n",
" ('same', '*'),\n",
" ('dude', '*'),\n",
" ('different', '*'),\n",
" ('*', 'you'),\n",
" ('*', \"won't\"),\n",
" ('*', 'let'),\n",
" ('*', 'all'),\n",
" ('*', 'the'),\n",
" ('*', 'bullshit'),\n",
" ('*', 'they'),\n",
" ('*', 'say'),\n",
" ('*', 'get'),\n",
" ('*', 'to'),\n",
" ('*', 'you'),\n",
" ('*', 'you'),\n",
" ('*', 'a'),\n",
" ('*', '10'),\n",
" ('*', \"i'm\"),\n",
" ('*', 'not'),\n",
" ('*', 'perfect'),\n",
" ('*', 'but'),\n",
" ('*', 'for'),\n",
" ('*', 'you'),\n",
" ('*', 'i'),\n",
" ('*', 'do'),\n",
" ('*', 'pretend'),\n",
" ('*', \"i've\"),\n",
" ('*', 'been'),\n",
" ('*', 'working'),\n",
" ('*', '2'),\n",
" ('*', 'to'),\n",
" ('*', '10'),\n",
" ('*', \"that's\"),\n",
" ('*', 'am'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'nobody'),\n",
" ('*', 'quite'),\n",
" ('*', 'like'),\n",
" ('*', 'him'),\n",
" ('*', 'put'),\n",
" ('*', 'in'),\n",
" ('*', 'the'),\n",
" ('*', 'time'),\n",
" ('*', 'like'),\n",
" ('*', 'him'),\n",
" ('*', \"i've\"),\n",
" ('*', 'been'),\n",
" ('*', 'grinding'),\n",
" ('*', 'for'),\n",
" ('*', 'a'),\n",
" ('*', 'while'),\n",
" ('*', 'now'),\n",
" ('*', \"it's\"),\n",
" ('*', 'feeling'),\n",
" ('*', 'like'),\n",
" ('*', 'my'),\n",
" ('*', 'time'),\n",
" ('*', 'is'),\n",
" ('*', 'finally'),\n",
" ('*', 'here'),\n",
" ('*', \"that's\"),\n",
" ('*', 'ideal'),\n",
" ('*', \"that's\"),\n",
" ('*', 'ideal'),\n",
" ('*', 'like'),\n",
" ('*', \"what's\"),\n",
" ('*', 'next'),\n",
" ('*', \"what's\"),\n",
" ('*', 'that'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'wanna'),\n",
" ('*', 'chill'),\n",
" ('drink', 'drink'),\n",
" ('*', 'smoke'),\n",
" ('*', 'fuck'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'god'),\n",
" ('*', 'damn'),\n",
" ('*', 'that'),\n",
" ('*', \"body's\"),\n",
" ('*', 'goals'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'somehow'),\n",
" ('*', 'you'),\n",
" ('*', 'hot'),\n",
" ('*', 'as'),\n",
" ('*', 'hell'),\n",
" ('*', 'but'),\n",
" ('*', 'still'),\n",
" ('*', 'cold'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', 'the'),\n",
" ('*', 'deal'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'you'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'the'),\n",
" ('*', 'only'),\n",
" ('*', 'one'),\n",
" ('*', 'you'),\n",
" ('*', 'knew'),\n",
" ('*', 'that'),\n",
" ('*', \"i've\"),\n",
" ('*', 'been'),\n",
" ('*', 'doing'),\n",
" ('*', 'great'),\n",
" ('*', 'i'),\n",
" ('*', 'guess'),\n",
" ('*', 'i'),\n",
" ('*', \"can't\"),\n",
" ('*', 'complain'),\n",
" ('*', 'i'),\n",
" ('*', \"don't\"),\n",
" ('*', 'think'),\n",
" ('*', 'about'),\n",
" ('*', 'yesterday'),\n",
" ('*', 'every'),\n",
" ('*', 'move'),\n",
" ('*', 'is'),\n",
" ('*', 'calculated'),\n",
" ('this', 'this'),\n",
" ('*', 'shit'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'no'),\n",
" ('*', 'guessing'),\n",
" ('*', 'game'),\n",
" ('*', 'only'),\n",
" ('*', 'time'),\n",
" ('*', \"i'm\"),\n",
" ('*', 'second'),\n",
" ('*', 'place'),\n",
" ('is', 'is'),\n",
" ('*', 'when'),\n",
" ('*', \"i'm\"),\n",
" ('*', 'at'),\n",
" ('*', 'my'),\n",
" ('*', 'second'),\n",
" ('*', 'place'),\n",
" ('*', 'made'),\n",
" ('*', '300'),\n",
" ('*', 'then'),\n",
" ('*', 'doubled'),\n",
" ('*', 'that'),\n",
" ('*', 'i'),\n",
" ('*', 'guess'),\n",
" ('*', 'you'),\n",
" ('*', 'call'),\n",
" ('*', 'that'),\n",
" ('*', '2nd'),\n",
" ('*', 'base'),\n",
" ('*', 'how'),\n",
" ('*', 'many'),\n",
" ('*', 'come'),\n",
" ('*', 'ups'),\n",
" ('*', 'until'),\n",
" ('*', \"it's\"),\n",
" ('*', 'destiny'),\n",
" ('*', 'how'),\n",
" ('*', 'many'),\n",
" ('*', 'come'),\n",
" ('*', 'ups'),\n",
" ('*', 'until'),\n",
" ('*', \"it's\"),\n",
" ('*', 'meant'),\n",
" ('*', 'to'),\n",
" ('*', 'be'),\n",
" ('*', 'i'),\n",
" ('*', 'hear'),\n",
" ('*', 'them'),\n",
" ('*', 'talking'),\n",
" ('*', 'that'),\n",
" ('*', \"don't\"),\n",
" ('*', 'get'),\n",
" ('*', 'to'),\n",
" ('me', 'me'),\n",
" ('*', 'that'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'shit'),\n",
" ('*', 'to'),\n",
" ('*', 'me'),\n",
" ('*', 'same'),\n",
" ('*', 'dude'),\n",
" ('*', 'different'),\n",
" ('*', 'dream'),\n",
" ('this', 'this'),\n",
" ('is', 'is'),\n",
" ('me', 'me'),\n",
" (\"ain't\", \"ain't\"),\n",
" ('nothing', 'nothing'),\n",
" ('pretty', 'pretty'),\n",
" ('but', 'but'),\n",
" ('her', 'her'),\n",
" ('face', 'face'),\n",
" ('money', 'money'),\n",
" ('hungry', 'hungry'),\n",
" ('nothing', 'nothing'),\n",
" ('skinny', 'skinny'),\n",
" ('but', 'but'),\n",
" ('her', 'her'),\n",
" ('waist', 'waist'),\n",
" ('*', 'she'),\n",
" ('*', 'come'),\n",
" ('*', 'from'),\n",
" ('*', 'money'),\n",
" ('*', 'but'),\n",
" ('*', 'she'),\n",
" ('*', 'spent'),\n",
" ('*', 'it'),\n",
" ('*', 'on'),\n",
" ('*', 'her'),\n",
" ('*', 'titties'),\n",
" ('*', 'and'),\n",
" ('*', 'her'),\n",
" ('*', 'face'),\n",
" ('*', 'if'),\n",
" ('*', 'you'),\n",
" ('*', 'really'),\n",
" ('*', 'wanna'),\n",
" ('*', 'know'),\n",
" ('*', 'these'),\n",
" ('*', 'women'),\n",
" ('*', 'man'),\n",
" ('*', 'they'),\n",
" ('*', 'come'),\n",
" ('*', 'and'),\n",
" ('*', 'go'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'one'),\n",
" ('*', 'minute'),\n",
" ('*', \"she's\"),\n",
" ('*', 'the'),\n",
" ('*', 'one'),\n",
" ('*', 'the'),\n",
" ('*', 'next'),\n",
" ('*', 'you'),\n",
" ('*', 'really'),\n",
" ('*', 'never'),\n",
" ('*', 'know'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'she'),\n",
" ('*', \"don't\"),\n",
" ('*', 'know'),\n",
" ('*', 'a'),\n",
" ('*', 'damn'),\n",
" ('*', 'thing'),\n",
" ('*', 'about'),\n",
" ('*', 'love'),\n",
" ('*', 'she'),\n",
" ('*', \"don't\"),\n",
" ('*', 'know'),\n",
" ('*', 'a'),\n",
" ('*', 'damn'),\n",
" ('*', 'thing'),\n",
" ('*', 'at'),\n",
" ('*', 'all'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'one'),\n",
" ('*', 'minute'),\n",
" ('*', \"she's\"),\n",
" ('*', 'the'),\n",
" ('*', 'one'),\n",
" ('*', 'the'),\n",
" ('*', 'next'),\n",
" ('*', 'you'),\n",
" ('*', 'really'),\n",
" ('*', 'never'),\n",
" ('*', 'know'),\n",
" ('*', 'like'),\n",
" ('*', \"what's\"),\n",
" ('*', 'next'),\n",
" ('*', \"what's\"),\n",
" ('*', 'that'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'i'),\n",
" ('*', 'just'),\n",
" ('*', 'wanna'),\n",
" ('*', 'chill'),\n",
" ('*', 'drink'),\n",
" ('*', 'smoke'),\n",
" ('*', 'fuck'),\n",
" ('*', \"'cause\"),\n",
" ('*', 'god'),\n",
" ('*', 'damn'),\n",
" ('*', 'that'),\n",
" ('*', \"body's\"),\n",
" ('*', 'goals'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'somehow'),\n",
" ('*', 'you'),\n",
" ('*', 'hot'),\n",
" ('*', 'as'),\n",
" ('*', 'hell'),\n",
" ('*', 'but'),\n",
" ('*', 'still'),\n",
" ('*', 'cold'),\n",
" ('*', 'as'),\n",
" ('*', 'fuck'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', 'the'),\n",
" ('*', 'deal'),\n",
" ('*', 'you'),\n",
" ('*', 'know'),\n",
" ('*', \"what's\"),\n",
" ('*', 'up'),\n",
" ('*', 'you'),\n",
" ('*', \"ain't\"),\n",
" ('*', 'the'),\n",
" ('*', 'only'),\n",
" ('*', 'one'),\n",
" ('*', 'girls'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'you'),\n",
" ('*', 'ready'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'uh'),\n",
" ('*', 'you'),\n",
" ('*', 'ready'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'you'),\n",
" ('*', 'ready'),\n",
" ('*', 'girls'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'girls'),\n",
" ('*', 'these'),\n",
" ('*', 'days'),\n",
" ('*', 'these'),\n",
" ('*', 'days42embed')]"
]
},
"execution_count": 108,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(zip(out[0].seqA, out[0].seqB))"
]
},
{
"cell_type": "code",
"execution_count": 122,
"id": "1f3fa3b6",
"metadata": {},
"outputs": [],
"source": [
"# make index from output back to seqA index\n",
"seqB_idx_map = {}\n",
"for n, b in enumerate(out[0].seqB):\n",
" if b == \"*\":\n",
" continue\n",
" seqB_idx_map[n] = len(seqB_idx_map)"
]
},
{
"cell_type": "code",
"execution_count": 112,
"id": "c8b35214",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"16 longest match streak\n"
]
}
],
"source": [
"# find longest consecutive match\n",
"best_match = None\n",
"running_start = None\n",
"for n, (a, b) in enumerate(zip(out[0].seqA, out[0].seqB)):\n",
" if a == b:\n",
" if running_start is None:\n",
" running_start = n\n",
" else:\n",
" if (\n",
" running_start is not None and \n",
" (best_match is None or n - running_start > best_match[1] - best_match[0])\n",
" ):\n",
" best_match = (running_start, n)\n",
" running_start = None\n",
"if (\n",
" running_start is not None and \n",
" (best_match is None or len(out[0].seqA) - running_start < best_match[1] - best_match[0])\n",
"):\n",
" best_match = (running_start, len(out[0].seqA))\n",
"print(best_match[1] - best_match[0], \"longest match streak\")"
]
},
{
"cell_type": "code",
"execution_count": 113,
"id": "6b61a0bb",
"metadata": {},
"outputs": [],
"source": [
"# TODO: try to extend match to everything separated mach by 2 mismatches\n",
"prelim_start_idx, prelim_end_idx = best_match\n",
"\n",
"# l = [a == b for a, b in zip(out[0].seqA, out[0].seqB)]\n",
"# front_part = \n",
"# first_match_idx = None\n",
"# last_match_idx = None\n",
"# idx_b = 0\n",
"# for a, b in zip(out[0].seqA, out[0].seqB):\n",
"# if b != \"*\":\n",
"# idx_b += 1\n",
"# if a == b != \"*\":\n",
"# last_match_idx = idx_b\n",
"# if first_match_idx is None:\n",
"# first_match_idx = idx_b\n",
" \n",
"start_idx, end_idx = prelim_start_idx, prelim_end_idx"
]
},
{
"cell_type": "code",
"execution_count": 123,
"id": "6c464c28",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\"This is me Ain't nothing pretty but her face Money hungry, nothing skinny but her waist\""
]
},
"execution_count": 123,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\" \".join(haystack_list[haystack_idx_map[seqB_idx_map[start_idx]]:haystack_idx_map[seqB_idx_map[end_idx]]])"
]
},
{
"cell_type": "code",
"execution_count": 115,
"id": "6b744e56",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"That ain't shit to me. Same dude different drink this is me this is me Ain't nothing pretty but her face. Money hungry, nothing skinny but her waist.\n"
]
}
],
"source": [
"print(tokens.plaintext)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "025cb42f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "b65cf409",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cdecb293",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}