"""Detect whether text contains the name of a popular artist.""" import re import string from pathlib import Path import numpy as np import wordfreq from unidecode import unidecode from suno_utils.worker.top_artists import TOP_ARTISTS def _strip_punctuation(text: str) -> str: exceptions = {"$", "?"} delenda = set(string.punctuation) - exceptions return text.translate({ord(c): "" for c in delenda}) def prep_text(text: str) -> str: """Preprocess artist names.""" steps = [ unidecode, str.lower, _strip_punctuation, str.strip, ] for step in steps: text = step(text) return text def _is_mostly_ascii(text: str) -> str: """Determine whether text is mostly ascii modulo a few accented chars.""" unidecoded_text = unidecode(text) if len(text) != len(unidecoded_text): return False else: return np.mean([t == u for (t, u) in zip(text, unidecoded_text, strict=True)]) > 0.75 # These will be normalized and post-processed below. MANUAL_BLOCKED_STRINGS = [ "Taylor", "Swift", "BTS", "the rolling stones", "周杰伦", "michael jackson", "beyonce", "dj snake", "lady gaga", "alice cooper", "arcade fire", "ariana grande", "the beatles", "ben harper", "billie holiday", "black eyed peas", "black sabbath", "blondie", "bob dylan", "bob marley", "boyz ii men", "bruce springsteen", "bryan adams", "bryan ferry", "carrie underwood", "cher", "chris cornell", "chuck berry", "the clash", "counting crows", "crowded house", "david bowie", "diana ross", "dire straits", "don henley", "donna summer", "dr dre", "ed sheeran", "ella fitzgerald", "elton john", "elvis presley", "elvis", "eminem", "fleetwood mac", "foo fighters", "foxy brown", "frank sinatra", "freddie mercury", "gang starr", "george benson", "george strait", "glen campbell", "gregory porter", "halsey", "ice cube", "iggy pop", "iron maiden", "isaac hayes", "james morrison", "janet jackson", "jessie j", "jimi hendrix", "jimmy buffett", "jimmy cliff", "john lee hooker", "john lennon", "johnny gill", "jonny lang", "joss stone", "justin bieber", "kaiser chiefs", "katy perry", "keane", "kendrick lamar", "the killers", "the kinks", "kip moore", "lana del rey", "led zeppelin", "lil wayne", "lionel richie", "lorde", "louis armstrong", "luke bryan", "marilyn manson", "maroon 5", "the mavericks", "maxi priest", "meat loaf", "metallica", "the moody blues", "nat king cole", "neil diamond", "nina simone", "nine inch nails", "pj harvey", "papa roach", "pearl jam", "phil collins", "pink floyd", "queens of the stone age", "quincy jones", "red hot chili peppers", "rihanna", "rob zombie", "robbie williams", "robert palmer", "rod stewart", "roxy music", "sabrina carpenter", "sam cooke", "scorpions", "selena gomez", "sergio mendes", "sex pistols", "slick rick", "the smashing pumpkins", "smokey robinson", "sonic youth", "spice girls", "steve earle", "steven tyler", "stevie wonder", "suzanne vega", "tears for fears", "the temptations", "tim mcgraw", "toby keith", "tom waits", "tori amos", "the tragically hip", "the velvet underground", "vince gill", "willie nelson", "hank williams jr", "britney spears", "billy joel", "miley cyrus", "drake", "kanye", "coldplay", "bruno mars", "post malone", "jay z", "jay-z", ] # these strings are ok even though they're contained in the artist name text file MANUAL_EXCEPTIONS = [ "airbag", "annemarie", "anouk", "avant", "blossoms", "bunbury", "ceylan", "doble", "eels", "epica", "freestyle", "ghazal", "kalimba", "kayou", "kofi", "latino", "lofi hip hop", "maska", "nazareth", "pesado", "phosphate", "pollo", "quincy", "shepherd", "solitario", "sophie", "sputnik", "tesla", "tori", "vacations", ] MIN_ARTIST_LENGTH = 4 def _filter_for_frequency(artists: list[str], threshold: float) -> list[str]: """Filter out all artists whose general word frequency is above a given threshold.""" return [artist for artist in artists if wordfreq.word_frequency(artist, lang="en") < threshold] def _define_BLOCKED_ARTISTS(): """Get list of all latin-alphabet-based artists not in dictionary of length >= 4""" # TODO: for some reason modal doesn't have this file under "/usr/share/dict/words" with open(Path(__file__).parent / "assets/words.py") as f: excluded_words = [(line.strip()) for line in f.readlines()] excluded_words += MANUAL_EXCEPTIONS top_ascii_artists = [artist for artist in TOP_ARTISTS if _is_mostly_ascii(artist)] prepped_artists = set(map(prep_text, top_ascii_artists)) prepped_excluded_words = set(map(prep_text, excluded_words)) all_blocked_artists = prepped_artists - prepped_excluded_words long_enough_artists = [artist for artist in all_blocked_artists if len(artist) >= MIN_ARTIST_LENGTH] rare_enough_artists = _filter_for_frequency(long_enough_artists, threshold=10**-6) return rare_enough_artists + [prep_text(s) for s in MANUAL_BLOCKED_STRINGS] BLOCKED_ARTISTS = _define_BLOCKED_ARTISTS() BLOCKED_ARTIST_REGEX = re.compile( "|".join([(rf"(? str | None: """If text contains a popular artist name, return it, else None.""" clean_text = prep_text(text) if match_ := BLOCKED_ARTIST_REGEX.search(clean_text): return match_.group() else: return None