main mt 1

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 3/28 In [2]: !date spotify_users = load_pickle('user_playlists.pickle') print("==> Finished loading the data.") !date Familiarize yourself with these data The variable spotify_users holds the data you'll need. It consists of a list of about 15,000 or so users: In [3]: print(f"`spotify_users`: type == {type(spotify_users)}, number of elem ents == {len(spotify_users):,}.") Each element of this list corresponds to a distinct user. Have a look at the user at position 2526 of this list: In [4]: pprint(spotify_users[2526]) Fri 26 Feb 2021 12:27:54 AM PST Opening pickle from './resource/asnlib/publicdata/user_playlists.pickl e' ... ==> Finished loading the data. Fri 26 Feb 2021 12:28:03 AM PST `spotify_users`: type == <class 'list'>, number of elements == 15,918. {'playlists': [{'name': 'Favoritas de la radio', 'tracks': [{'artist': 'Vico C', 'title': 'Desahogo'}, {'artist': 'Vico C', 'title': 'El Bueno, El Malo Y El Feo (The Good, ' 'The Bad & The Ugly) - Feat. Tego ' 'Calderón And Eddie Dee'}, {'artist': 'Vico C', 'title': 'Quieren'}, {'artist': 'Vico C', 'title': "Vamonos Po' Encima"}]}, {'name': 'Starred', 'tracks': [{'artist': 'Vico C', 'title': 'El'}, {'artist': 'Strike 3', 'title': 'Enamorado De Ti'}, {'artist': 'Strike 3', 'title': 'Es Por T i'}]}, {'name': 'Two', 'tracks': [{'artist': 'Walk the Moon', 'title': 'Quesa dilla'}, {'artist': 'Two Door Cinema Club', 'title': 'Sleep Alone'}, {'artist': 'Two Door Cinema Club', 'title': 'Something Good Can Work'}, {'artist': 'Two Door Cinema Club', 'title': 'Sun'}]}], 'user_id': '22c5af0c50b557327894d0c9ea6aa5fa'}

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 6/28 In [8]: # Test cell 0: `mt1_ex0_count_playlists` (1 point) ### BEGIN HIDDEN TESTS global_overwrite = False def tracks_iterator(users, offsets= False ): for i, u in enumerate(users): for j, p in enumerate(u['playlists']): for k, t in enumerate(p['tracks']): if offsets: yield u, p, t, i, j, k else : yield u, p, t def randomly_error(threshold=0.05): from random import random return random() < threshold def mt1_ex0__gen_soln(): print(f"The Spotify dataset consists of {count_playlists(spotify_u sers):,} playlists in total.") #!date #mt1_ex0__gen_soln() #!date ### END HIDDEN TESTS #from testing_tools import mt1_ex0__check #print("Testing...") assert count_playlists(spotify_users) == 231_844 from testing_tools import mt1_ex0__check for trial in range(250): mt1_ex0__check(count_playlists) print(" \n (Passed!)") (Passed!)

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 9/28 Part B: Data cleaning Unfortunately, artist names are encoded in a messy fashion. Here are some examples: The artist "Jay-Z" is written as "Jay-Z" and "JAY Z" , with several other variations having different capitalization. Worse, there is no consistent standard for encoding multiple artists who worked together on a song. For example, here is how several of Jay-Z's collaborations appear: 'Alicia Keys ft. Jay-Z' ( 'ft.' used as an artist-separator) 'A-Trak x Kanye x Jay-Z' ( ' x ' used as an artist-separator) 'JAY Z Featuring Beyoncé' (variation on "Jay-Z" and yet another variation on "featuring" to separate artists) 'Jay-Z Featuring Beyoncé Knowles' (Beyoncé's last name included in this variation) 'Jay-Z/Kanye West/Lil Wayne/T.I.' (... you get the idea ...) 'Jay Z (Dr. Dre, Rakim, & Truth Hurts)' 'Young Jeezy Ft. Jay-Z & Fat Joe' 'Lil Wayne Drake Jay-Z And Gif Majorz' (spaces used ambiguously: there are four artists in this example!) 'Timbaland & Magoo feat Jay-Z' 'OutKast/Jay-Z/Killer Mike' 'Jay-Z Ft.Rihanna And Kanye West' 'Pat Benetar vs. Beyonce vs. 3OH!3 Feat. Britney Spears, Christina Aguilera, & M.I.A.' ("Benatar" is misspelled as 'Benetar' ) 'jay z with the roots. s' (yes, excess spaces between . and trailing s are real) It is difficult to design a robust algorithm to extract individual artist names. Instead, let's use the following approximate algorithm, given an artist-string as input. 1. Lowercase: First, convert all characters to lowercase. 2. Space-equivalents: Next, convert any hyphen ( '-' ), period ( '.' ), question mark ( '?' ), exclamation point ( '!' ), and underscores ( '_' ) into a space character. 3. Separators: Then split the string, treating the following patterns as artist-name separators: A. All of the following words, but only when there are spaces both before and after: 'and' , 'with' , 'ft' , 'feat' , 'featuring' , 'vs' , and 'x' . B. All of the following symbols: '/' (forward slash), '&' (ampersand), comma ( ',' ), semicolon ( ';' ), and each enclosing parenthesis or bracket ( '(' , ')' , '[' , ']' , '{' , '}' ) 4. Whitespace compression: Lastly, for any artist name-string following the above separation steps, strip out any preceding and trailing whitespace and collapse multiple consecutive whitespace characters into a single space. When applying this algorithm, we'll perform steps 1-4 in the exact same sequence as shown above. ⟹ ⟹ ⟹ ⟹ ⟹ ⟹ ⟹ ⟹

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 12/28 In [15]: # Test cell: `mt1_ex2_extract_artists` (2 points) ### BEGIN HIDDEN TESTS def mt1_ex2__gen_soln(fn_base="artist_translation_table", fn_ext="pick le", overwrite= False ): from testing_tools import file_exists, load_pickle, save_pickle fn = f" {fn_base} . {fn_ext} " if file_exists(fn) and not overwrite: print(f"' {fn} ' exists; skipping...") else : # not file_exists(fn) or overwrite print(f"' {fn} ' does not exist or needs to be overwritten; gene rating...") artist_translation_table = {} for _, _, t in tracks_iterator(spotify_users): artist_translation_table[t['artist']] = extract_artists(t ['artist']) save_pickle(artist_translation_table, fn) !date mt1_ex2__gen_soln(overwrite= False or global_overwrite) !date ### END HIDDEN TESTS from testing_tools import mt1_ex2__check print("Testing...") for trial in range(250): mt1_ex2__check(extract_artists) extract_artists__passed = True print(" \n (Passed!)") Sample results for Exercise 2: artist_translation_table If you had a working solution to Exercise 2, then in principle you could use it to normalize and separate the artist names. We have precomputed these translations for you, for every artist name that appears in the data; run the cell below to load a name-translation table, stored in the variable, artist_translation_table . Read and run this cell even if you skipped or otherwise did not complete Exercise 2. Fri 26 Feb 2021 12:28:04 AM PST 'artist_translation_table.pickle' exists; skipping... Fri 26 Feb 2021 12:28:04 AM PST Testing... (Passed!)

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 15/28 In [20]: # Demo cell ex3_your_output = extract_playlists(ex0_demo_users) print("=== Your output ===") pprint(ex3_your_output) assert all(a == b for a, b in zip(ex3_your_output, ex3_demo_output)), "Your output does not match the demo output!" print(" \n (Your output matches the demo output — so far, so good!)") In [21]: # Test cell: `mt1_ex3_extract_playlists` (2 points) ### BEGIN HIDDEN TESTS def mt1_ex3__gen_soln(fn_base="simple_playlists", fn_ext="pickle", ove rwrite= False ): from testing_tools import file_exists, load_pickle, save_pickle fn = f" {fn_base} . {fn_ext} " if file_exists(fn) and not overwrite: print(f"' {fn} ' exists; skipping...") else : # not file_exists(fn) or overwrite print(f"' {fn} ' does not exist or needs to be overwritten; gene rating...") simple_playlists = extract_playlists(spotify_users) save_pickle(simple_playlists, fn) !date mt1_ex3__gen_soln(overwrite= False or global_overwrite) !date ### END HIDDEN TESTS from testing_tools import mt1_ex3__check print("Testing...") for trial in range(250): mt1_ex3__check(extract_playlists) extract_playlists__passed = True print(" \n (Passed!)") === Your output === [{'André Rieu'}, {'Lucio Battisti', 'Alicia Keys ft. Jay-Z', 'The Police'}, {'U2'}] (Your output matches the demo output — so far, so good!) Fri 26 Feb 2021 12:28:04 AM PST 'simple_playlists.pickle' exists; skipping... Fri 26 Feb 2021 12:28:05 AM PST Testing... (Passed!)

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 18/28 For example, recall how this function worked in the case where "words" are receipts and the individual letters are itemsets. Furthermore, simply calling the default set on one receipt creates an itemset: In [27]: make_itemsets(['hello', 'world']) To use make_itemsets for our problem, we need to create a function that is compatible with the requirements of the make_set argument. That is your next task. Exercise 4: normalize_artist_set (2 points) Complete the function, normalize_artist_set(artist_set) , where artist_set is a Python set of unprocessed artist names. It should return a Python set of normalized artist names, per Exercise 2. For instance, normalize_artist_set({'Alicia Keys ft. Jay-Z', 'Lucio Battisti', 'The Polic e'}) should return {'the police', 'lucio battisti', 'alicia keys', 'jay z'} Note: You may reuse your function from Exercise 2, if you are confident it is bug-free; otherwise, we recommend using the precomputed values in artist_translation_table . In [28]: def normalize_artist_set(artist_set): ### BEGIN SOLUTION global artist_translation_table # not strictly necessary, but self -documenting output_set = set() for a in artist_set: output_set |= artist_translation_table[a] return output_set ### END SOLUTION In [29]: # Demo cell: normalize_artist_set({'Alicia Keys ft. Jay-Z', 'Lucio Battisti', 'The Police'}) # expected output: `{'alicia keys', 'jay z', 'lucio battisti', 'th e police'}` Out[27]: [{'e', 'h', 'l', 'o'}, {'d', 'l', 'o', 'r', 'w'}] Out[29]: {'alicia keys', 'jay z', 'lucio battisti', 'the police'}

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 21/28 Exercise 5: get_artist_counts (1 point) For the Notebook 2 analysis, we also needed a way to count in how many receipts each item occurred. That's your next task. Given a collection of artist itemsets, complete the function get_artist_counts(itemsets) so that it returns a dictionary-like object with artist names as keys and the number of occurrences as values. For example, suppose you start with these three itemsets: itemsets = [{'alicia keys', 'jay z', 'lucio battisti', 'the police'}, {'u2', 'the police'}, {'jay z'}] Then get_artist_counts(itemsets) should return: {'alicia keys': 1, 'jay z': 2, 'lucio battisti': 1, 'the police': 2, 'u2': 1} Note: By "dictionary-like," we mean either a conventional Python dictionary or a collections.defaultdict , as you prefer. Hint: Recall update_item_counts from Notebook 2, which we've provided again in the code cell below. In [32]: def update_item_counts(item_counts, itemset): for a in itemset: item_counts[a] += 1 def get_artist_counts(itemsets): ### BEGIN SOLUTION from collections import defaultdict counts = defaultdict(int) for s in itemsets: update_item_counts(counts, s) return counts ### END SOLUTION In [33]: # Demo cell: itemsets = [{'alicia keys', 'jay z', 'lucio battisti', 'the police'}, {'u2', 'the police'}, {'jay z'}] get_artist_counts(itemsets) Out[33]: defaultdict(int, {'alicia keys': 1, 'lucio battisti': 1, 'jay z': 2, 'the police': 2, 'u2': 1})

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 22/28 In [34]: # Test cell: `mt1_ex5_get_artist_counts` (1 point) ### BEGIN HIDDEN TESTS def mt1_ex5__gen_soln(fn_base="artist_counts", fn_ext="pickle", overwr ite= False ): from testing_tools import file_exists, load_pickle, save_pickle fn = f" {fn_base} . {fn_ext} " if file_exists(fn) and not overwrite: print(f"' {fn} ' exists; skipping...") else : # not file_exists(fn) or overwrite print(f"' {fn} ' does not exist or needs to be overwritten; gene rating...") artist_sets = load_pickle('normalized_artist_sets.pickle') artist_counts = get_artist_counts(artist_sets) save_pickle(artist_counts, fn) !date mt1_ex5__gen_soln(overwrite= False or global_overwrite) !date ### END HIDDEN TESTS from testing_tools import mt1_ex5__check print("Testing...") for trial in range(250): mt1_ex5__check(get_artist_counts) get_artist_counts__passed = True print(" \n (Passed!)") Sample results for Exercise 5: artist_counts If you had a working solution to Exercise 5, then in principle you could run get_artist_counts(artist_itemsets) to count the number of occurrences of all artists. Instead, we have precomputed these for you; run the cell below to load it into the object, artist_counts . Read and run this cell even if you skipped or otherwise did not complete Exercise 5. Fri 26 Feb 2021 12:28:10 AM PST 'artist_counts.pickle' exists; skipping... Fri 26 Feb 2021 12:28:11 AM PST Testing... (Passed!)

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 23/28 In [35]: artist_counts = load_pickle('artist_counts.pickle') print("Examples:") for a in ['lady gaga', 'fats domino', 'kishi bashi']: print(f"* Artist ' {a} ' appears in {artist_counts[a]:,} playlist s.") Part E: A simple artist-recommender system We now have all the pieces we need to build a recommender system to help users find artists they might like, building on Notebook 2's pairwise association-rule miner. However, we'll need a modified procedure. Why? Recall how many artists there are (run the cell below): In [36]: print(f'The dataset has {len(artist_counts):,} artists! (After name no rmalization per Exercise 2.)') That's a lot! So rather than finding all association rules, let's use the following procedure instead. 1. First, suppose a user has given us the name of one artist they already like. Call that the root artist . 2. Filter all playlists to only those containing the root artist. Call these the root playlists (or root itemsets ). 3. For each root playlist, remove any artists that are "uncommon," based on a given threshold. However, do not remove the root artist; those should always be kept, whether common or not. Call these resulting playlists the pruned playlists. 4. Run the pairwise association rule miner on these pruned playlists, which should be smaller and thus faster to process, and report the top result(s). For your last exercise, we'll give you code for Step 2 and need you to combine it with Step 3. We will provide the rest, and if your procedure works, you'll be able to try it out! Filtering step. Here is code we are providing for Step 2 of this proposed recommender algorithm (filter playlists). In [37]: def filter_itemsets(root_item, itemsets): return [s for s in itemsets if root_item in s] Opening pickle from './resource/asnlib/publicdata/artist_counts.pickl e' ... Examples: * Artist 'lady gaga' appears in 5,121 playlists. * Artist 'fats domino' appears in 327 playlists. * Artist 'kishi bashi' appears in 427 playlists. The dataset has 258,036 artists! (After name normalization per Exercis e 2.)

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 24/28 Here is a demo of filter_itemsets , which generates "root playlists" for the artist, " Kishi Bashi (https://www.kishibashi.com/)." Pop-up Video / Behind The Lyrics trivia: At the time of this exam (Spring 2021), Kishi Bashi lives in Athens, Georgia, USA, about 90-minutes or so outside Atlanta! In [38]: root_playlists_for_kishi_bashi = filter_itemsets('kishi bashi', artist _itemsets) print(f"Found {len(root_playlists_for_kishi_bashi)} playlists containi ng 'kishi bashi.'") print("Example:", root_playlists_for_kishi_bashi[2]) Found 427 playlists containing 'kishi bashi.' Example: {'hozier', 'the new pornographers', 'plushgun', 'the smashing pumpkins', 'rockabye baby', 'discovery', "chris o'brien", 'matt nathan son', 'ed sheeran', 'the xx', 'lisa hannigan', 'first aid kit', 'clap your hands say yeah', 'kishi bashi', 'stars'}

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 25/28 Exercise 6: prune_itemsets (2 points) Complete the function, def prune_itemsets(root_item, itemsets, item_counts, min_count): ... so that it implements Step 2 and Step 3 of the recommender. That is, the inputs are: root_item : The root item (i.e., the root artist name) itemsets : A collection of itemsets item_counts : A pre-tabulated count of how many times each item appears in an itemset min_count : The minimum number of itemsets in which an item should appear to be considered a recommendation Your function should return the playlists pruned as follows: 1. Filter the itemsets to only those containing root_item . The resulting itemsets are the filtered itemsets. 2. For each filtered itemset, remove any item where item_counts[a] < min_count . However, do not remove root_item , regardless of its count. 3. The resulting itemsets are the pruned itemsets. Discard any pruned itemsets that contain only the root item. Return the remaining pruned itemsets as a Python list of sets. Note 0: Although the procedure above is written as though your function will modify its input arguments, it must not do so. Use copies as needed instead. The test cell will not pass if you modify the input arguments. Note 1: You can return pruned itemsets in any order. (So if the test cell does not pass, it is not because it assumes results in a particular order.) Example. Suppose the itemsets and item counts are given as follows: In [39]: ex6_demo_itemsets = [{'alicia keys', 'jay z', 'lucio battisti', 'the p olice'}, {'u2', 'the police'}, {'jay z'}] ex6_demo_item_counts = {'alicia keys': 1, 'jay z': 2, 'lucio battist i': 1, 'the police': 2, 'u2': 1} Then prune_itemsets('the police', ex6_demo_itemsets, ex6_demo_item_counts, 2) will end up returning a list with just one itemset, [{'the police', 'jay z'}] . That's because only two itemsets have 'the police' in them, and of those, only one has at least one item whose count exceeds min_count=2 .

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 26/28 In [40]: def prune_itemsets(root_item, itemsets, item_counts, min_count): ### BEGIN SOLUTION filtered_itemsets = filter_itemsets(root_item, itemsets) pruned_itemsets = [] for s in filtered_itemsets: s_pruned = set() for x in s: if item_counts[x] >= min_count or x == root_item: s_pruned |= {x} if len(s_pruned) >= 2: pruned_itemsets.append(s_pruned) return pruned_itemsets ### END SOLUTION In [41]: # Demo cell: prune_itemsets('the police', ex6_demo_itemsets, ex6_demo_item_counts, 2) Out[41]: [{'jay z', 'the police'}]

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 27/28 In [42]: # Test cell: `mt1_ex6_prune_itemsets` (2 points) ### BEGIN HIDDEN TESTS def mt1_ex6__gen_soln(fn_base="pruned_playlists", root='kishi bashi', threshold=1000, fn_ext="pickle", overwrite= False ): from testing_tools import file_exists, load_pickle, save_pickle fn = f" {fn_base} --{root.replace(' ', '-')}-- {threshold} . {fn_ext} " if file_exists(fn) and not overwrite: print(f"' {fn} ' exists; skipping...") else : # not file_exists(fn) or overwrite print(f"' {fn} ' does not exist or needs to be overwritten; gene rating...") artist_itemsets = load_pickle('normalized_artist_sets.pickle') artist_counts = load_pickle('artist_counts.pickle') pruned_playlists = prune_itemsets(root, artist_itemsets, artis t_counts, threshold) save_pickle(pruned_playlists, fn) !date mt1_ex6__gen_soln(overwrite= False or global_overwrite) !date ### END HIDDEN TESTS from testing_tools import mt1_ex6__check print("Testing...") for trial in range(250): mt1_ex6__check(prune_itemsets) prune_itemsets__passed = True print(" \n (Passed!)") Fin! If you passed the preceding exercise, then you have all the pieces necessary to try your recommendation algorithm! It is optional to do so, but if you have any time left, pick your favorite artist (assuming they are in the dataset) and see if you get reasonable results. Otherwise, you’ve reached the end of this problem. Don’t forget to restart and run all cells again to make sure your code works when running all code cells in sequence; and make sure your work passes the submission process. Good luck! Fri 26 Feb 2021 12:28:14 AM PST 'pruned_playlists--kishi-bashi--1000.pickle' exists; skipping... Fri 26 Feb 2021 12:28:14 AM PST Testing... (Passed!)

11/28/23, 8:41 PM main file:///Users/dannie/Downloads/pmt1-sample-solutions-su21/problem24-sample-solutions.html 28/28 In [43]: assert prune_itemsets__passed == True , "Are you sure you passed Exerci se 6?" # `recommend` implements the complete recommender algorithm def recommend(root_artist, conf=0.2, min_count=1000, verbose= True ): from cse6040nb2 import find_assoc_rules, print_rules global artist_itemsets, artist_counts print("Pruning...") pruned_playlists = prune_itemsets(root_artist, artist_itemsets, ar tist_counts, min_count) num_artists = sum(len(p) for p in pruned_playlists) print(" \t ", len(pruned_playlists), "itemsets remain with", num_art ists, "artists.") print("Finding association rules...") rules = find_assoc_rules(pruned_playlists, conf) rules = {(a, b): c for (a, b), c in rules.items() if a == root_art ist} print(" \t ", len(rules), f"rules of the form `conf(' {root_artist} ' => x) >= {conf} ") print(f" \n === Our top recommendations for ' {root_artist} ' ===") print_rules(rules, limit=20) # DEMO: 'kishi bashi' produces some spurious results because # both "Of Monsters and Men" and "Mumford and Sons" are # erroneously split into two. recommend('kishi bashi') Pruning... 411 itemsets remain with 19530 artists. Finding association rules... 20 rules of the form `conf('kishi bashi' => x) >= 0.2 === Our top recommendations for 'kishi bashi' === conf(kishi bashi => alt j) = 0.265 conf(kishi bashi => passion pit) = 0.258 conf(kishi bashi => men) = 0.255 conf(kishi bashi => of monsters) = 0.255 conf(kishi bashi => vampire weekend) = 0.231 conf(kishi bashi => bon iver) = 0.229 conf(kishi bashi => grizzly bear) = 0.224 conf(kishi bashi => the lumineers) = 0.219 conf(kishi bashi => two door cinema club) = 0.219 conf(kishi bashi => the xx) = 0.219 conf(kishi bashi => m83) = 0.219 conf(kishi bashi => the shins) = 0.219 conf(kishi bashi => first aid kit) = 0.217 conf(kishi bashi => sons) = 0.214 conf(kishi bashi => mumford) = 0.212 conf(kishi bashi => the black keys) = 0.212 conf(kishi bashi => grouplove) = 0.212 conf(kishi bashi => lana del rey) = 0.204 conf(kishi bashi => imagine dragons) = 0.204 conf(kishi bashi => phantogram) = 0.202