[extractor/servus] Rewrite extractor (#6036)

Closes #1076, closes #4240, closes #2748, closes #1045, closes #1498
Authored by: FrankZ85, Ashish0804, StefanLobbenmeier

Co-authored-by: FrankZ85 <43293037+FrankZ85@users.noreply.github.com>

yt_dlp/extractor/servus.py

@@ -1,11 +1,13 @@
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
+    ExtractorError,
     float_or_none,
+    format_field,
     int_or_none,
+    join_nonempty,
+    traverse_obj,
+    unescapeHTML,
     unified_timestamp,
-    urlencode_postdata,
-    url_or_none,
 )
 
 
@@ -15,32 +17,41 @@ class ServusIE(InfoExtractor):
                         (?:www\.)?
                         (?:
                             servus\.com/(?:(?:at|de)/p/[^/]+|tv/videos)|
-                            (?:servustv|pm-wissen)\.com/videos
+                            (?:servustv|pm-wissen)\.com/(?:[^/]+/)?v(?:ideos)?
                         )
-                        /(?P<id>[aA]{2}-\w+|\d+-\d+)
+                        /(?P<id>[aA]{2}-?\w+|\d+-\d+)
                     '''
     _TESTS = [{
-        # new URL schema
+        # URL schema v3
-        'url': 'https://www.servustv.com/videos/aa-1t6vbu5pw1w12/',
+        'url': 'https://www.servustv.com/natur/v/aa-28bycqnh92111/',
-        'md5': '60474d4c21f3eb148838f215c37f02b9',
         'info_dict': {
-            'id': 'AA-1T6VBU5PW1W12',
+            'id': 'AA-28BYCQNH92111',
             'ext': 'mp4',
-            'title': 'Die Grünen aus Sicht des Volkes',
+            'title': 'Klettersteige in den Alpen',
-            'alt_title': 'Talk im Hangar-7 Voxpops Gruene',
+            'description': 'md5:25e47ddd83a009a0f9789ba18f2850ce',
-            'description': 'md5:1247204d85783afe3682644398ff2ec4',
             'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 62.442,
+            'duration': 2823,
-            'timestamp': 1605193976,
+            'timestamp': 1655752333,
-            'upload_date': '20201112',
+            'upload_date': '20220620',
-            'series': 'Talk im Hangar-7',
+            'series': 'Bergwelten',
-            'season': 'Season 9',
+            'season': 'Season 11',
-            'season_number': 9,
+            'season_number': 11,
-            'episode': 'Episode 31 - September 14',
+            'episode': 'Episode 8 - Vie Ferrate – Klettersteige in den Alpen',
-            'episode_number': 31,
+            'episode_number': 8,
-        }
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        'url': 'https://www.servustv.com/natur/v/aa-1xg5xwmgw2112/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.servustv.com/natur/v/aansszcx3yi9jmlmhdc1/',
+        'only_matching': True,
     }, {
-        # old URL schema
+        # URL schema v2
+        'url': 'https://www.servustv.com/videos/aa-1t6vbu5pw1w12/',
+        'only_matching': True,
+    }, {
+        # URL schema v1
         'url': 'https://www.servus.com/de/p/Die-Gr%C3%BCnen-aus-Sicht-des-Volkes/AA-1T6VBU5PW1W12/',
         'only_matching': True,
     }, {
@@ -60,85 +71,65 @@ class ServusIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url).upper()
 
-        token = self._download_json(
-            'https://auth.redbullmediahouse.com/token', video_id,
-            'Downloading token', data=urlencode_postdata({
-                'grant_type': 'client_credentials',
-            }), headers={
-                'Authorization': 'Basic SVgtMjJYNEhBNFdEM1cxMTpEdDRVSkFLd2ZOMG5IMjB1NGFBWTBmUFpDNlpoQ1EzNA==',
-            })
-        access_token = token['access_token']
-        token_type = token.get('token_type', 'Bearer')
-
         video = self._download_json(
-            'https://sparkle-api.liiift.io/api/v1/stv/channels/international/assets/%s' % video_id,
+            'https://api-player.redbull.com/stv/servus-tv?timeZone=Europe/Berlin',
-            video_id, 'Downloading video JSON', headers={
+            video_id, 'Downloading video JSON', query={'videoId': video_id})
-                'Authorization': '%s %s' % (token_type, access_token),
+        if not video.get('videoUrl'):
-            })
+            self._report_errors(video)
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            video['videoUrl'], video_id, 'mp4', m3u8_id='hls')
 
-        formats = []
+        season = video.get('season')
-        thumbnail = None
-        for resource in video['resources']:
-            if not isinstance(resource, dict):
-                continue
-            format_url = url_or_none(resource.get('url'))
-            if not format_url:
-                continue
-            extension = resource.get('extension')
-            type_ = resource.get('type')
-            if extension == 'jpg' or type_ == 'reference_keyframe':
-                thumbnail = format_url
-                continue
-            ext = determine_ext(format_url)
-            if type_ == 'dash' or ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(
-                    format_url, video_id, mpd_id='dash', fatal=False))
-            elif type_ == 'hls' or ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif extension == 'mp4' or ext == 'mp4':
-                formats.append({
-                    'url': format_url,
-                    'format_id': type_,
-                    'width': int_or_none(resource.get('width')),
-                    'height': int_or_none(resource.get('height')),
-                })
-
-        attrs = {}
-        for attribute in video['attributes']:
-            if not isinstance(attribute, dict):
-                continue
-            key = attribute.get('fieldKey')
-            value = attribute.get('fieldValue')
-            if not key or not value:
-                continue
-            attrs[key] = value
-
-        title = attrs.get('title_stv') or video_id
-        alt_title = attrs.get('title')
-        description = attrs.get('long_description') or attrs.get('short_description')
-        series = attrs.get('label')
-        season = attrs.get('season')
-        episode = attrs.get('chapter')
-        duration = float_or_none(attrs.get('duration'), scale=1000)
         season_number = int_or_none(self._search_regex(
             r'Season (\d+)', season or '', 'season number', default=None))
+        episode = video.get('chapter')
         episode_number = int_or_none(self._search_regex(
             r'Episode (\d+)', episode or '', 'episode number', default=None))
 
         return {
             'id': video_id,
-            'title': title,
+            'title': video.get('title'),
-            'alt_title': alt_title,
+            'description': self._get_description(video_id) or video.get('description'),
-            'description': description,
+            'thumbnail': video.get('poster'),
-            'thumbnail': thumbnail,
+            'duration': float_or_none(video.get('duration')),
-            'duration': duration,
+            'timestamp': unified_timestamp(video.get('currentSunrise')),
-            'timestamp': unified_timestamp(video.get('lastPublished')),
+            'series': video.get('label'),
-            'series': series,
             'season': season,
             'season_number': season_number,
             'episode': episode,
             'episode_number': episode_number,
             'formats': formats,
+            'subtitles': subtitles,
         }
+
+    def _get_description(self, video_id):
+        info = self._download_json(
+            f'https://backend.servustv.com/wp-json/rbmh/v2/media_asset/aa_id/{video_id}?fieldset=page',
+            video_id, fatal=False)
+
+        return join_nonempty(*traverse_obj(info, (
+            ('stv_short_description', 'stv_long_description'),
+            {lambda x: unescapeHTML(x.replace('\n\n', '\n'))})), delim='\n\n')
+
+    def _report_errors(self, video):
+        playability_errors = traverse_obj(video, ('playabilityErrors', ...))
+        if not playability_errors:
+            raise ExtractorError('No videoUrl and no information about errors')
+
+        elif 'FSK_BLOCKED' in playability_errors:
+            details = traverse_obj(video, ('playabilityErrorDetails', 'FSK_BLOCKED'), expected_type=dict)
+            message = format_field(''.join((
+                format_field(details, 'minEveningHour', ' from %02d:00'),
+                format_field(details, 'maxMorningHour', ' to %02d:00'),
+                format_field(details, 'minAge', ' (Minimum age %d)'),
+            )), None, 'Only available%s') or 'Blocked by FSK with unknown availability'
+
+        elif 'NOT_YET_AVAILABLE' in playability_errors:
+            message = format_field(
+                video, (('playabilityErrorDetails', 'NOT_YET_AVAILABLE', 'availableFrom'), 'currentSunrise'),
+                'Only available from %s') or 'Video not yet available with unknown availability'
+
+        else:
+            message = f'Video unavailable: {", ".join(playability_errors)}'
+
+        raise ExtractorError(message, expected=True)