From e04a1ff92e015bf431486d1fbcc8b243a92bfc71 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 5 Sep 2021 10:27:49 +0530 Subject: [PATCH] [soundcloud] Retry playlist pages on `502` error Closes #872 --- yt_dlp/extractor/soundcloud.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/yt_dlp/extractor/soundcloud.py b/yt_dlp/extractor/soundcloud.py index a9ccb7a8b..c7078ece6 100644 --- a/yt_dlp/extractor/soundcloud.py +++ b/yt_dlp/extractor/soundcloud.py @@ -23,6 +23,8 @@ from ..utils import ( int_or_none, KNOWN_EXTENSIONS, mimetype2ext, + network_exceptions, + remove_end, parse_qs, str_or_none, try_get, @@ -662,7 +664,7 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudIE): 'entries': self._entries(base_url, playlist_id), } - def _entries(self, base_url, playlist_id): + def _entries(self, url, playlist_id): # Per the SoundCloud documentation, the maximum limit for a linked partitioning query is 200. # https://developers.soundcloud.com/blog/offset-pagination-deprecated query = { @@ -671,12 +673,25 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudIE): 'offset': 0, } + retries = self.get_param('extractor_retries', 3) - next_href = base_url for i in itertools.count(): - response = self._download_json( - next_href, playlist_id, - 'Downloading track page %s' % (i + 1), query=query, headers=self._HEADERS) + attempt, last_error = -1, None + while attempt < retries: + attempt += 1 + if last_error: + self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'), playlist_id) + try: + response = self._download_json( + url, playlist_id, query=query, headers=self._HEADERS, + note='Downloading track page %s%s' % (i + 1, f' (retry #{attempt})' if attempt else '')) + break + except ExtractorError as e: + # Downloading page may result in intermittent 502 HTTP error + # See https://github.com/yt-dlp/yt-dlp/issues/872 + if attempt >= retries or not isinstance(e.cause, compat_HTTPError) or e.cause.code != 502: + raise + last_error = str(e.cause or e.msg) def resolve_entry(*candidates): for cand in candidates: @@ -692,7 +707,7 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudIE): for e in response['collection'] or []: yield resolve_entry(e, e.get('track'), e.get('playlist')) - next_href = response.get('next_href') + url = response.get('next_href') query.pop('offset', None)