[spotify:show] Fix extractor

Closes #3768

yt_dlp/extractor/spotify.py

@@ -1,12 +1,15 @@
+import functools
 import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    OnDemandPagedList,
     clean_podcast_url,
     float_or_none,
     int_or_none,
     strip_or_none,
+    traverse_obj,
     try_get,
     unified_strdate,
 )
@@ -25,7 +28,7 @@ class SpotifyBaseIE(InfoExtractor):
         self._ACCESS_TOKEN = self._download_json(
             'https://open.spotify.com/get_access_token', None)['accessToken']
 
-    def _call_api(self, operation, video_id, variables):
+    def _call_api(self, operation, video_id, variables, **kwargs):
         return self._download_json(
             'https://api-partner.spotify.com/pathfinder/v1/query', video_id, query={
                 'operationName': 'query' + operation,
@@ -35,7 +38,8 @@ class SpotifyBaseIE(InfoExtractor):
                         'sha256Hash': self._OPERATION_HASHES[operation],
                     },
                 })
-            }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN})['data']
+            }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN},
+            **kwargs)['data']
 
     def _extract_episode(self, episode, series):
         episode_id = episode['id']
@@ -143,22 +147,25 @@ class SpotifyShowIE(SpotifyBaseIE):
         },
         'playlist_mincount': 36,
     }
+    _PER_PAGE = 100
+
+    def _fetch_page(self, show_id, page=0):
+        return self._call_api('ShowEpisodes', show_id, {
+            'limit': 100,
+            'offset': page * self._PER_PAGE,
+            'uri': f'spotify:show:{show_id}',
+        }, note=f'Downloading page {page + 1} JSON metadata')['podcast']
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
-        podcast = self._call_api('ShowEpisodes', show_id, {
-            'limit': 1000000000,
-            'offset': 0,
-            'uri': 'spotify:show:' + show_id,
-        })['podcast']
-        podcast_name = podcast.get('name')
-
-        entries = []
-        for item in (try_get(podcast, lambda x: x['episodes']['items']) or []):
-            episode = item.get('episode')
-            if not episode:
-                continue
-            entries.append(self._extract_episode(episode, podcast_name))
+        first_page = self._fetch_page(show_id)
+
+        def _entries(page):
+            podcast = self._fetch_page(show_id, page) if page else first_page
+            yield from map(
+                functools.partial(self._extract_episode, series=podcast.get('name')),
+                traverse_obj(podcast, ('episodes', 'items', ..., 'episode')))
 
         return self.playlist_result(
-            entries, show_id, podcast_name, podcast.get('description'))
+            OnDemandPagedList(_entries, self._PER_PAGE),
+            show_id, first_page.get('name'), first_page.get('description'))