Major bug fixes

settings.py

@@ -2,6 +2,7 @@ class Settings:
     DEFAULT_DOWNLOAD_DIRECTORY = 'E:\\Spotify\\'
     ARTIST_IMAGES_SUB_DIR = '_Artists'
     PLAYLIST_METADATA_SUB_DIR = '_Playlists'
+    CATEGORY_METADATA_SUB_DIR = '_Categories'
     GLOBALS_SAVE_FILE = '_downloaded_store.json'
     FULL_DOWNLOAD_RECURISVE_LIMIT = 0x4000
     FULL_DOWNLOAD_THREAD_LIMIT = 50

spotify_mass_download.py

@@ -150,24 +150,36 @@ def full_download(download_dir: str, identifier: str, recursive_artist: bool=Fal
         console.error(f'Full download exception: {ex}')
 
 
-def download_all_categories_playlists(download_meta_data_only=True):
+def download_category_playlists(category_id, category_index, category_ids, download_meta_data_only):
+    playlist_ids = scraper.get_category_playlist_ids(category_id)
+    random.shuffle(playlist_ids)
+    for playlist_index, playlist_id in enumerate(playlist_ids):
+        console.log(f'Scraping playlist data from playlist {playlist_id} ({playlist_index + 1}/{len(playlist_ids)}) from category {category_id} ({category_index + 1}/{len(category_ids)})')
+        try:
+            playlist = scraper.get_playlist(playlist_id)
+            playlist.export_to_file()
+            if not download_meta_data_only:
+                full_download(f'{settings.DEFAULT_DOWNLOAD_DIRECTORY}', identifier=playlist.href, thread_count=15)
+        except Exception as ex:
+            console.error(f'Scraping categories exception: {ex}')
+
+
+def download_all_categories_playlists(download_meta_data_only=True, query:str=''):
     client.refresh_tokens()
     os.makedirs(f'{settings.DEFAULT_DOWNLOAD_DIRECTORY}/{settings.PLAYLIST_METADATA_SUB_DIR}/', exist_ok=True)
-    console.log(f'Scraping playlists from all categories')
-    category_ids = scraper.get_categories_ids()
-    random.shuffle(category_ids)
-    for category_index, category_id in enumerate(category_ids):
-        console.log(f'Scraping playlists from category {category_id} ({category_index + 1}/{len(category_ids)})')
+    console.log(f'Scraping playlists from "{query}" categories')
+    categories = scraper.get_categories_full(query=query)
+    threads = []
+    random.shuffle(categories)
+    for category_index, category in enumerate(categories):
+        console.log(f'Scraping playlists from category {category.name} ({category_index + 1}/{len(categories)})')
+        #category.download_metadata(scraper=scraper)
         try:
-            playlist_ids = scraper.get_category_playlist_ids(category_id)
-            for playlist_index, playlist_id in enumerate(playlist_ids):
-                console.log(f'Scraping playlist data from playlist {playlist_id} ({playlist_index + 1}/{len(playlist_ids)}) from category {category_id} ({category_index + 1}/{len(category_ids)})')
-                try:
-                    playlist = scraper.get_playlist(playlist_id)
-                    playlist.export_to_file()
-                    if not download_meta_data_only:
-                        full_download(f'{settings.DEFAULT_DOWNLOAD_DIRECTORY}', identifier=playlist.href, thread_count=15)
-                except Exception as ex:
-                    console.error(f'Scraping categories exception: {ex}')
+            thread = Thread(target=download_category_playlists, args=(category.spotify_id, category_index, categories, download_meta_data_only))
+            thread.start()
+            threads.append(thread)
+            #download_category_playlists(category_id, category_index=category_index, category_ids=category_ids, download_meta_data_only=download_meta_data_only)
         except Exception as ex:
                     console.error(f'Scraping categories exception: {ex}')
+
+    [x.join() for x in threads]

spotify_scraper.py

@@ -155,17 +155,21 @@ class SpotifyScraper:
         return playlist_ids
 
     def get_category_playlists(self, category_id: str, limit:int=50, offset:int=0) -> str:
-        return self.get(f'https://api.spotify.com/v1/browse/categories/{category_id}/playlists/?limit={limit}&offset={offset}').json()
+        data = self.get(f'https://api.spotify.com/v1/browse/categories/{category_id}/playlists/?limit={limit}&offset={offset}').json()
+        return data
 
     def get_categories(self, limit=50) -> str:
-        return self.get(f'https://api.spotify.com/v1/browse/categories/?limit={limit}').json()
+        return self.get(f'https://api.spotify.com/v1/browse/categories/?limit={limit}&country=IL').json()
 
-    def get_categories_ids(self, limit=50) -> str:
+    def get_categories_full(self, limit=50, query:str='') -> list[SpotifyCategory]:
         categories = self.get_categories()
-        ids = []
-        for category in categories['categories']['items']:
-            ids.append(category['id'])
-        return ids
+        categories_data = []
+        os.makedirs(f'{settings.DEFAULT_DOWNLOAD_DIRECTORY}/{settings.CATEGORY_METADATA_SUB_DIR}/', exist_ok=True)
+        for category_json in categories['categories']['items']:
+            if not query or query.lower() in category_json['name'].lower(): 
+                category = SpotifyCategory(category_json)
+                categories_data.append(category)
+        return categories_data
 
     def get_playlist_data(self, playlist_id: str) -> str:
         return self.get(f'https://api.spotify.com/v1/playlists/{playlist_id}').json()

spotify_utils.py

@@ -3,6 +3,40 @@ import base64
 from config import *
 
 
+class SpotifyCategory:
+    name = ''
+    spotify_id = ''
+    playlist_ids = ''
+    thumbnail_href = ''
+
+    def __init__(self, category_data=None):
+        self.name = category_data['name']
+        self.spotify_id = category_data['id']
+        if len(category_data['icons']) > 0:
+            self.thumbnail_href = category_data['icons'][0]['url']
+
+    def download_metadata(self, scraper):
+
+        thumbail_b64 = ''
+        if self.thumbnail_href:
+            thumbail_b64 = base64.b64encode( requests.get(self.thumbnail_href).content ).decode()
+
+        try:
+            self.playlist_ids = scraper.get_category_playlist_ids(category_id=self.spotify_id)
+        except:
+            self.playlist_ids = []
+
+        data = {
+            'name': self.name,
+            'spotify_id': self.spotify_id,
+            'thumbnail_b64': thumbail_b64,
+            'playlist_ids': self.playlist_ids,
+        }
+
+        with open(f'{settings.DEFAULT_DOWNLOAD_DIRECTORY}/{settings.CATEGORY_METADATA_SUB_DIR}/{self.spotify_id}.category', 'w') as f:
+            f.write(json.dumps(data))
+
+
 class SpotifyAlbum:
     title = ''
     thumbnail_href = ''

templates/index.html

@@ -54,9 +54,13 @@
             <div class="single-line"><label>Thread Limit: </label><input type="number" name="thread-count" value="5"></div>
             </div></div>
         </form>
+        <form id="category-download-form" action="/actions/download/categories" method="POST" target="dummy-frame">
+            <input type="text" id="query" name="query" placeholder="Category query (e.g. 'pop')" style="width:30em;">
+            <input type="submit" value="Download"/><br>
+        </form>
     </div>
     <div>
-        <div style="display:inline-flex;align-items:baseline;"><label>Show settings: </label><input id="autoscroll" type="checkbox" checked onchange="toggle_settings_visibility();"></div>
+        <div style="display:inline-flex;align-items:baseline;"><label>Show settings: </label><input type="checkbox" checked onchange="toggle_settings_visibility();"></div>
         <div id="settings-container" style="display:block;" name="Settings">
             <form action="/settings/" method="POST" target="dummy-frame" style="width:100%;">
                 <input name="download-dir" placeholder="Download directory" value="{{settings.DEFAULT_DOWNLOAD_DIRECTORY}}">

webgui.py

@@ -30,9 +30,12 @@ def actions_download():
         return str(ex)
 
 
-@app.route('/actions/download/categories')
+@app.route('/actions/download/categories', methods=['POST'])
 def actions_download_categories():
-    download_all_categories_playlists(download_meta_data_only=False)
+    query = request.form.get('query')
+    if not query:
+        query = ''
+    download_all_categories_playlists(download_meta_data_only=False, query=query)
 
 
 @app.route('/info/console/')