Merge pull request #28 from Keeper-of-the-Keys/soundcloud-optimize

Soundcloud optimize

src/gpodder/__init__.py

@@ -50,9 +50,9 @@
 # This metadata block gets parsed by setup.py - use single quotes only
 __tagline__ = 'Media and podcast aggregator'
 __author__ = 'Thomas Perl <[email protected]>'
-__version__ = '4.8.0'
-__date__ = '2019-10-31'
-__relname__ = 'Yoreh'
+__version__ = '4.9.0'
+__date__ = '2019-12-10'
+__relname__ = 'Tal'
 __copyright__ = '© 2005-2019 Thomas Perl and the gPodder Team'
 __license__ = 'ISC / GPLv3 or later'
 __url__ = 'http://gpodder.org/'

src/gpodder/plugins/soundcloud.py

@@ -91,16 +91,19 @@ def get_metadata(url):
 class SoundcloudUser(object):
     def __init__(self, username):
         self.username = username
+        self.cache = {}
 
     def get_user_info(self):
         global CONSUMER_KEY
         key = ':'.join((self.username, 'user_info'))
 
-        json_url = 'https://api.soundcloud.com/users/%s.json?consumer_key=%s' % (self.username, CONSUMER_KEY)
-        logger.debug('get_user_info url: %s', json_url)
-        user_info = json.loads(util.urlopen(json_url).read().decode('utf-8'))
+        if key not in self.cache:
+            json_url = 'https://api.soundcloud.com/users/%s.json?consumer_key=%s' % (self.username, CONSUMER_KEY)
+            logger.debug('get_user_info url: %s', json_url)
+            user_info = json.loads(util.urlopen(json_url).read().decode('utf-8'))
+            self.cache[key] = user_info
 
-        return user_info
+        return self.cache[key]
 
     def get_coverart(self):
         user_info = self.get_user_info()
@@ -114,7 +117,7 @@ def get_username(self):
         user_info = self.get_user_info()
         return user_info.get('username', None)
 
-    def get_tracks(self, feed):
+    def get_tracks(self, feed, channel):
         """Get a generator of tracks from a SC user
 
         The generator will give you a dictionary for every
@@ -133,6 +136,15 @@ def get_tracks(self, feed):
         json_tracks = json.loads(util.urlopen(json_url).read().decode('utf-8'))
         tracks = [track for track in json_tracks if track['streamable'] or track['downloadable']]
 
+        self.cache['episodes'] = { episode.guid:
+                                   { "filesize": episode.file_size,
+                                     "filetype": episode.mime_type,
+                                   } for episode in channel.episodes
+                                 }
+
+        read_from_cache = 0
+        logger.debug('%d Episodes in database for Soundcloud:%s', len(self.cache['episodes']), self.username)
+
         for track in tracks:
             # Prefer stream URL (MP3), fallback to download URL
             base_url = track.get('stream_url') if track['streamable'] else track.get('download_url')
@@ -142,9 +154,14 @@ def get_tracks(self, feed):
                 logger.debug('Skipping track with no base_url')
                 continue
 
-            logger.debug('track in tracks url: %s', url)
+            track_guid = track.get('permalink', track.get('id'))
 
-            filesize, filetype, filename = get_metadata(url)
+            if track_guid not in self.cache['episodes']:
+                filesize, filetype, filename = get_metadata(url)
+            else:
+                filesize = self.cache['episodes'][track_guid]['filesize']
+                filetype = self.cache['episodes'][track_guid]['filetype']
+                read_from_cache += 1
 
             yield {
                 'title': track.get('title', track.get('permalink')) or ('Unknown track'),
@@ -153,10 +170,13 @@ def get_tracks(self, feed):
                 'url': url,
                 'file_size': int(filesize),
                 'mime_type': filetype,
-                'guid': track.get('permalink', track.get('id')),
+                'guid': track_guid,
                 'published': soundcloud_parsedate(track.get('created_at', None)),
+                'total_time': int(track.get('duration') / 1000),
             }
 
+        logger.debug('Read %d episodes from %d cached episodes', read_from_cache, len(self.cache['episodes']))
+
 
 class SoundcloudFeed(object):
     def __init__(self, username):
@@ -191,7 +211,7 @@ def get_new_episodes(self, channel):
         return self._get_new_episodes(channel, 'tracks')
 
     def _get_new_episodes(self, channel, track_type):
-        tracks = [t for t in self.sc_user.get_tracks(track_type)]
+        tracks = [t for t in self.sc_user.get_tracks(track_type, channel)]
 
         existing_guids = [episode.guid for episode in channel.episodes]
         seen_guids = [track['guid'] for track in tracks]