Merge branch 'feature/autoscrape_playlist_metadata' into dev

config.py

@@ -24,6 +24,7 @@ VERIFY_SSL = True
 FULL_DOWNLOAD_RECURISVE_LIMIT = 0x4000
 FULL_DOWNLOAD_THREAD_LIMIT = 50
 VERBOSE_OUTPUTS = False
+AUTO_DOWNLOAD_PLAYLIST_METADATA = True
 
 DEFAULT_DOWNLOAD_DIRECTORY = 'music'
 ARTIST_IMAGES_SUB_DIR = '_Artists'

spotify_mass_download.py

@@ -159,8 +159,7 @@ def download_all_categories_playlists(download_meta_data_only=True):
                 console.log(f'Scraping playlist data from playlist {playlist_id} ({playlist_index + 1}/{len(playlist_ids)}) from category {category_id} ({category_index + 1}/{len(category_ids)})')
                 try:
                     playlist = scraper.get_playlist(playlist_id)
-                    with open(f'{DEFAULT_DOWNLOAD_DIRECTORY}/{PLAYLIST_METADATA_SUB_DIR}/{playlist.spotify_id}.playlist', 'w') as f:
+                    playlist.export_to_file()
-                        f.write(playlist.export())
                     if not download_meta_data_only:
                         full_download(f'{DEFAULT_DOWNLOAD_DIRECTORY}', identifier=playlist.href)
                 except Exception as ex:

spotify_scraper.py

@@ -60,7 +60,11 @@ class SpotifyScraper:
             tracks += playlist_data['items']
         if len(tracks) != int(playlist_data['total']):
             print(f'Warning: track count does not match! {len(tracks)} != {int(playlist_data["tracks"]["total"])}')
-        return [SpotifyTrack(track_data) for track_data in tracks]
+        spotify_tracks = [SpotifyTrack(track_data) for track_data in tracks]
+        if AUTO_DOWNLOAD_PLAYLIST_METADATA:
+            playlist = SpotifyPlaylist(playlist_id, spotify_tracks, self.get_playlist_data(playlist_id))
+            playlist.export_to_file()
+        return spotify_tracks
 
     def scrape_album(self, album_id: str):
         return self._client.get(f'https://api.spotify.com/v1/albums/{album_id}').json()
@@ -139,6 +143,8 @@ class SpotifyScraper:
             category_playlists_json = self.get_category_playlists(category_id, limit=50, offset=current_offset)
             has_next = category_playlists_json['playlists']['next'] is not None
             for playlist in category_playlists_json['playlists']['items']:
+                if not playlist:
+                    continue
                 playlist_ids.append(playlist['id'])
         return playlist_ids
 
@@ -155,7 +161,10 @@ class SpotifyScraper:
             ids.append(category['id'])
         return ids
 
+    def get_playlist_data(self, playlist_id: str) -> str:
+        return self.get(f'https://api.spotify.com/v1/playlists/{playlist_id}').json()
+
     def get_playlist(self, playlist_id: str) -> str:
-        playlist_data = self.get(f'https://api.spotify.com/v1/playlists/{playlist_id}').json()
+        playlist_data = self.get_playlist_data(playlist_id)
         tracks = self.scrape_playlist_tracks(playlist_id)
         return SpotifyPlaylist(spotify_id=playlist_id, tracks=tracks, data=playlist_data)

spotify_utils.py

@@ -220,6 +220,10 @@ class SpotifyPlaylist:
             'track_ids': [track.spotify_id for track in self.tracks]
             }
         return json.dumps(data)
+    
+    def export_to_file(self) -> None:
+        with open(f'{DEFAULT_DOWNLOAD_DIRECTORY}/{PLAYLIST_METADATA_SUB_DIR}/{self.spotify_id}.playlist', 'w') as f:
+            f.write(self.export())
 
     @property
     def href(self):