diff --git a/yt_dlp/extractor/spotify.py b/yt_dlp/extractor/spotify.py index a2068a1b6..fef8d8dd2 100644 --- a/yt_dlp/extractor/spotify.py +++ b/yt_dlp/extractor/spotify.py @@ -1,12 +1,15 @@ +import functools import json import re from .common import InfoExtractor from ..utils import ( + OnDemandPagedList, clean_podcast_url, float_or_none, int_or_none, strip_or_none, + traverse_obj, try_get, unified_strdate, ) @@ -25,7 +28,7 @@ class SpotifyBaseIE(InfoExtractor): self._ACCESS_TOKEN = self._download_json( 'https://open.spotify.com/get_access_token', None)['accessToken'] - def _call_api(self, operation, video_id, variables): + def _call_api(self, operation, video_id, variables, **kwargs): return self._download_json( 'https://api-partner.spotify.com/pathfinder/v1/query', video_id, query={ 'operationName': 'query' + operation, @@ -35,7 +38,8 @@ class SpotifyBaseIE(InfoExtractor): 'sha256Hash': self._OPERATION_HASHES[operation], }, }) - }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN})['data'] + }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN}, + **kwargs)['data'] def _extract_episode(self, episode, series): episode_id = episode['id'] @@ -143,22 +147,25 @@ class SpotifyShowIE(SpotifyBaseIE): }, 'playlist_mincount': 36, } + _PER_PAGE = 100 + + def _fetch_page(self, show_id, page=0): + return self._call_api('ShowEpisodes', show_id, { + 'limit': 100, + 'offset': page * self._PER_PAGE, + 'uri': f'spotify:show:{show_id}', + }, note=f'Downloading page {page + 1} JSON metadata')['podcast'] def _real_extract(self, url): show_id = self._match_id(url) - podcast = self._call_api('ShowEpisodes', show_id, { - 'limit': 1000000000, - 'offset': 0, - 'uri': 'spotify:show:' + show_id, - })['podcast'] - podcast_name = podcast.get('name') - - entries = [] - for item in (try_get(podcast, lambda x: x['episodes']['items']) or []): - episode = item.get('episode') - if not episode: - continue - entries.append(self._extract_episode(episode, podcast_name)) + first_page = self._fetch_page(show_id) + + def _entries(page): + podcast = self._fetch_page(show_id, page) if page else first_page + yield from map( + functools.partial(self._extract_episode, series=podcast.get('name')), + traverse_obj(podcast, ('episodes', 'items', ..., 'episode'))) return self.playlist_result( - entries, show_id, podcast_name, podcast.get('description')) + OnDemandPagedList(_entries, self._PER_PAGE), + show_id, first_page.get('name'), first_page.get('description'))