[viu:ott] Fix extractor (see desc)

* add language_flag_id query param * add support for premium account (untested since I dont have a premium account) * support entire series Code from: https://github.com/blackjack4494/youtube-dlc/pull/211 https://github.com/ytdl-org/youtube-dl/pull/15182 https://github.com/ytdl-org/youtube-dl/pull/26775 Fixes: https://github.com/yt-dlp/yt-dlp/issues/219 https://github.com/ytdl-org/youtube-dl/issues/27946 https://github.com/ytdl-org/youtube-dl/issues/27863 https://github.com/ytdl-org/youtube-dl/issues/27812 https://github.com/ytdl-org/youtube-dl/issues/27464 https://github.com/ytdl-org/youtube-dl/issues/26788 https://github.com/blackjack4494/yt-dlc/issues/136 Possibly also fixes (untested): https://github.com/ytdl-org/youtube-dl/issues/16992 https://github.com/ytdl-org/youtube-dl/issues/26701 Co-authored by: lkho, pukkandan
4 years ago · baa5873942
parent c6ce815461
commit baa5873942
1 changed files with 140 additions and 10 deletions
--- a/yt_dlp/extractor/viu.py
+++ b/yt_dlp/extractor/viu.py
@ -1,16 +1,21 @@
 # coding: utf-8
 from __future__ import unicode_literals

+import json
 import re

 from .common import InfoExtractor
 from ..compat import (
    compat_kwargs,
    compat_str,
+    compat_urlparse,
+    compat_urllib_request,
 )
 from ..utils import (
    ExtractorError,
    int_or_none,
+    smuggle_url,
+    unsmuggle_url,
 )


@ -168,7 +173,8 @@ class ViuPlaylistIE(ViuBaseIE):

 class ViuOTTIE(InfoExtractor):
    IE_NAME = 'viu:ott'
-    _VALID_URL = r'https?://(?:www\.)?viu\.com/ott/(?P<country_code>[a-z]{2})/[a-z]{2}-[a-z]{2}/vod/(?P<id>\d+)'
+    _NETRC_MACHINE = 'viu'
+    _VALID_URL = r'https?://(?:www\.)?viu\.com/ott/(?P<country_code>[a-z]{2})/(?P<lang_code>[a-z]{2}-[a-z]{2})/vod/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.viu.com/ott/sg/en-us/vod/3421/The%20Prime%20Minister%20and%20I',
        'info_dict': {
@ -179,6 +185,7 @@ class ViuOTTIE(InfoExtractor):
        },
        'params': {
            'skip_download': 'm3u8 download',
+            'noplaylist': True,
        },
        'skip': 'Geo-restricted to Singapore',
    }, {
@ -191,6 +198,19 @@ class ViuOTTIE(InfoExtractor):
        },
        'params': {
            'skip_download': 'm3u8 download',
+            'noplaylist': True,
+        },
+        'skip': 'Geo-restricted to Hong Kong',
+    }, {
+        'url': 'https://www.viu.com/ott/hk/zh-hk/vod/68776/%E6%99%82%E5%B0%9A%E5%AA%BD%E5%92%AA',
+        'playlist_count': 12,
+        'info_dict': {
+            'id': '3916',
+            'title': '時尚媽咪',
+        },
+        'params': {
+            'skip_download': 'm3u8 download',
+            'noplaylist': False,
        },
        'skip': 'Geo-restricted to Hong Kong',
    }]
@ -201,9 +221,51 @@ class ViuOTTIE(InfoExtractor):
        'TH': 4,
        'PH': 5,
    }
+    _LANGUAGE_FLAG = {
+        'zh-hk': 1,
+        'zh-cn': 2,
+        'en-us': 3,
+    }
+    _user_info = None
+
+    def _detect_error(self, response):
+        code = response.get('status', {}).get('code')
+        if code > 0:
+            message = try_get(response, lambda x: x['status']['message'])
+            raise ExtractorError('%s said: %s (%s)' % (
+                self.IE_NAME, message, code), expected=True)
+        return response['data']
+
+    def _raise_login_required(self):
+        raise ExtractorError(
+            'This video requires login. '
+            'Specify --username and --password or --netrc (machine: %s) '
+            'to provide account credentials.' % self._NETRC_MACHINE,
+            expected=True)
+
+    def _login(self, country_code, video_id):
+        if not self._user_info:
+            username, password = self._get_login_info()
+            if username is None or password is None:
+                return
+
+            data = self._download_json(
+                compat_urllib_request.Request(
+                    'https://www.viu.com/ott/%s/index.php' % country_code, method='POST'),
+                video_id, 'Logging in', errnote=False, fatal=False,
+                query={'r': 'user/login'},
+                data=json.dumps({
+                    'username': username,
+                    'password': password,
+                    'platform_flag_label': 'web',
+                }).encode())
+            self._user_info = self._detect_error(data)['user']
+
+        return self._user_info

    def _real_extract(self, url):
-        country_code, video_id = re.match(self._VALID_URL, url).groups()
+        url, idata = unsmuggle_url(url, {})
+        country_code, lang_code, video_id = re.match(self._VALID_URL, url).groups()

        query = {
            'r': 'vod/ajax-detail',
@ -223,20 +285,88 @@ class ViuOTTIE(InfoExtractor):
        if not video_data:
            raise ExtractorError('This video is not available in your region.', expected=True)

-        stream_data = self._download_json(
-            'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code,
-            video_id, 'Downloading stream info', query={
+        series_id = video_data.get('series_id')
+        if not self._downloader.params.get('noplaylist') and not idata.get('force_noplaylist'):
+            self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % series_id)
+            series = product_data.get('series', {})
+            product = series.get('product')
+            if product:
+                entries = []
+                for entry in sorted(product, key=lambda x: int_or_none(x.get('number', 0))):
+                    item_id = entry.get('product_id')
+                    if not item_id:
+                        continue
+                    item_id = compat_str(item_id)
+                    entries.append(self.url_result(
+                        smuggle_url(
+                            'http://www.viu.com/ott/%s/%s/vod/%s/' % (country_code, lang_code, item_id),
+                            {'force_noplaylist': True}),  # prevent infinite recursion
+                        'ViuOTT',
+                        item_id,
+                        entry.get('synopsis', '').strip()))
+
+                return self.playlist_result(entries, series_id, series.get('name'), series.get('description'))
+
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
+
+        duration_limit = False
+        query = {
            'ccs_product_id': video_data['ccs_product_id'],
-            }, headers={
+            'language_flag_id': self._LANGUAGE_FLAG.get(lang_code.lower()) or '3',
+        }
+        headers = {
            'Referer': url,
-                'Origin': re.search(r'https?://[^/]+', url).group(0),
-            })['data']['stream']
+            'Origin': url,
+        }
+        try:
+            stream_data = self._download_json(
+                'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code,
+                video_id, 'Downloading stream info', query=query, headers=headers)
+            stream_data = self._detect_error(stream_data)['stream']
+        except (ExtractorError, KeyError):
+            stream_data = None
+            if video_data.get('user_level', 0) > 0:
+                user = self._login(country_code, video_id)
+                if user:
+                    query['identity'] = user['identity']
+                    stream_data = self._download_json(
+                        'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code,
+                        video_id, 'Downloading stream info', query=query, headers=headers)
+                    stream_data = self._detect_error(stream_data).get('stream')
+                else:
+                    # preview is limited to 3min for non-members
+                    # try to bypass the duration limit
+                    duration_limit = True
+                    query['duration'] = '180'
+                    stream_data = self._download_json(
+                        'https://d1k2us671qcoau.cloudfront.net/distribute_web_%s.php' % country_code,
+                        video_id, 'Downloading stream info', query=query, headers=headers)
+                    try:
+                        stream_data = self._detect_error(stream_data)['stream']
+                    except (ExtractorError, KeyError): # if still not working, give up
+                        self._raise_login_required()
+
+        if not stream_data:
+            raise ExtractorError('Cannot get stream info', expected=True)

        stream_sizes = stream_data.get('size', {})
        formats = []
        for vid_format, stream_url in stream_data.get('url', {}).items():
            height = int_or_none(self._search_regex(
                r's(\d+)p', vid_format, 'height', default=None))
+
+            # bypass preview duration limit
+            if duration_limit:
+                stream_url = compat_urlparse.urlparse(stream_url)
+                query = dict(compat_urlparse.parse_qsl(stream_url.query, keep_blank_values=True))
+                time_duration = int_or_none(video_data.get('time_duration'))
+                query.update({
+                    'duration': time_duration if time_duration > 0 else '9999999',
+                    'duration_start': '0',
+                })
+                stream_url = stream_url._replace(query=compat_urlparse.urlencode(query)).geturl()
+
            formats.append({
                'format_id': vid_format,
                'url': stream_url,