From 30bd1c16c8e2a767246cc115f12c1d2e99e2f8ec Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 29 Oct 2015 19:44:26 +0100 Subject: [PATCH 1/5] [adobetv] use api for extraction and add support specific language videos --- youtube_dl/extractor/adobetv.py | 60 +++++++++++++-------------------- 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py index 5e43adc51..3d25caad6 100644 --- a/youtube_dl/extractor/adobetv.py +++ b/youtube_dl/extractor/adobetv.py @@ -1,23 +1,26 @@ from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..utils import ( parse_duration, unified_strdate, str_to_int, + int_or_none, float_or_none, ISO639Utils, ) class AdobeTVIE(InfoExtractor): - _VALID_URL = r'https?://tv\.adobe\.com/watch/[^/]+/(?P[^/]+)' + _VALID_URL = r'https?://tv\.adobe\.com/(?:(?Pfr|de|es|jp)/)?watch/(?P[^/]+)/(?P[^/]+)' _TEST = { 'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/', 'md5': '9bc5727bcdd55251f35ad311ca74fa1e', 'info_dict': { - 'id': 'quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop', + 'id': '10981', 'ext': 'mp4', 'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop', 'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311', @@ -29,46 +32,31 @@ class AdobeTVIE(InfoExtractor): } def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - player = self._parse_json( - self._search_regex(r'html5player:\s*({.+?})\s*\n', webpage, 'player'), - video_id) - - title = player.get('title') or self._search_regex( - r'data-title="([^"]+)"', webpage, 'title') - description = self._og_search_description(webpage) - thumbnail = self._og_search_thumbnail(webpage) + language, show_urlname, urlname = re.match(self._VALID_URL, url).groups() + if not language: + language = 'en' - upload_date = unified_strdate( - self._html_search_meta('datepublished', webpage, 'upload date')) - - duration = parse_duration( - self._html_search_meta('duration', webpage, 'duration') or - self._search_regex( - r'Runtime:\s*(\d{2}:\d{2}:\d{2})', - webpage, 'duration', fatal=False)) - - view_count = str_to_int(self._search_regex( - r'
\s*Views?:\s*([\d,.]+)\s*
', - webpage, 'view count')) + video_data = self._download_json( + 'http://tv.adobe.com/api/v4/episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname), + urlname)['data'][0] formats = [{ - 'url': source['src'], - 'format_id': source.get('quality') or source['src'].split('-')[-1].split('.')[0] or None, - 'tbr': source.get('bitrate'), - } for source in player['sources']] + 'url': source['url'], + 'format_id': source.get('quality_level') or source['url'].split('-')[-1].split('.')[0] or None, + 'width': int_or_none(source.get('width')), + 'height': int_or_none(source.get('height')), + 'tbr': int_or_none(source.get('video_data_rate')), + } for source in video_data['videos']] self._sort_formats(formats) return { - 'id': video_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'upload_date': upload_date, - 'duration': duration, - 'view_count': view_count, + 'id': str(video_data['id']), + 'title': video_data['title'], + 'description': video_data.get('description'), + 'thumbnail': video_data.get('thumbnail'), + 'upload_date': unified_strdate(video_data.get('start_date')), + 'duration': parse_duration(video_data.get('duration')), + 'view_count': str_to_int(video_data.get('playcount')), 'formats': formats, } From 402ca40c9d0b20f1d04a0035b4cda0cc1184c689 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 29 Oct 2015 19:55:04 +0100 Subject: [PATCH 2/5] [adobetv] extract AdobeTVVideo info from json directly --- youtube_dl/extractor/adobetv.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py index 3d25caad6..383c89485 100644 --- a/youtube_dl/extractor/adobetv.py +++ b/youtube_dl/extractor/adobetv.py @@ -10,6 +10,7 @@ from ..utils import ( int_or_none, float_or_none, ISO639Utils, + determine_ext, ) @@ -79,28 +80,25 @@ class AdobeTVVideoIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - - webpage = self._download_webpage(url, video_id) - - player_params = self._parse_json(self._search_regex( - r'var\s+bridge\s*=\s*([^;]+);', webpage, 'player parameters'), - video_id) + video_data = self._download_json(url + '?format=json', video_id) formats = [{ + 'format_id': '%s-%s' % (determine_ext(source['src']), source.get('height')), 'url': source['src'], - 'width': source.get('width'), - 'height': source.get('height'), - 'tbr': source.get('bitrate'), - } for source in player_params['sources']] + 'width': int_or_none(source.get('width')), + 'height': int_or_none(source.get('height')), + 'tbr': int_or_none(source.get('bitrate')), + } for source in video_data['sources']] + self._sort_formats(formats) # For both metadata and downloaded files the duration varies among # formats. I just pick the max one duration = max(filter(None, [ float_or_none(source.get('duration'), scale=1000) - for source in player_params['sources']])) + for source in video_data['sources']])) subtitles = {} - for translation in player_params.get('translations', []): + for translation in video_data.get('translations', []): lang_id = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium']) if lang_id not in subtitles: subtitles[lang_id] = [] @@ -112,8 +110,9 @@ class AdobeTVVideoIE(InfoExtractor): return { 'id': video_id, 'formats': formats, - 'title': player_params['title'], - 'description': self._og_search_description(webpage), + 'title': video_data['title'], + 'description': video_data.get('description'), + 'thumbnail': video_data['video'].get('poster'), 'duration': duration, 'subtitles': subtitles, } From 9a605c8859e5ecf164719b890ea62b76afb0b874 Mon Sep 17 00:00:00 2001 From: remitamine Date: Thu, 29 Oct 2015 20:00:27 +0100 Subject: [PATCH 3/5] [adobetv] add support for show and channel extraction --- youtube_dl/extractor/__init__.py | 2 + youtube_dl/extractor/adobetv.py | 72 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 6318ac4a2..d4b42dc25 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -6,6 +6,8 @@ from .academicearth import AcademicEarthCourseIE from .addanime import AddAnimeIE from .adobetv import ( AdobeTVIE, + AdobeTVShowIE, + AdobeTVChannelIE, AdobeTVVideoIE, ) from .adultswim import AdultSwimIE diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py index 383c89485..d0bfafa45 100644 --- a/youtube_dl/extractor/adobetv.py +++ b/youtube_dl/extractor/adobetv.py @@ -62,6 +62,78 @@ class AdobeTVIE(InfoExtractor): } +class AdobeTVPlaylistBaseIE(InfoExtractor): + def _parse_page_data(self, page_data): + return [self.url_result(self._get_element_url(element_data)) for element_data in page_data] + + def _extract_playlist_entries(self, url, display_id): + page = self._download_json(url, display_id) + entries = self._parse_page_data(page['data']) + for page_num in range(2, page['paging']['pages'] + 1): + entries.extend(self._parse_page_data( + self._download_json(url + '&page=%d' % page_num, display_id)['data'])) + return entries + + +class AdobeTVShowIE(AdobeTVPlaylistBaseIE): + _VALID_URL = r'https?://tv\.adobe\.com/(?:(?Pfr|de|es|jp)/)?show/(?P[^/]+)' + + _TEST = { + 'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost', + 'info_dict': { + 'id': '36', + 'title': 'The Complete Picture with Julieanne Kost', + 'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27', + }, + 'playlist_mincount': 136, + } + + def _get_element_url(self, element_data): + return element_data['urls'][0] + + def _real_extract(self, url): + language, show_urlname = re.match(self._VALID_URL, url).groups() + if not language: + language = 'en' + query = 'language=%s&show_urlname=%s' % (language, show_urlname) + + show_data = self._download_json( + 'http://tv.adobe.com/api/v4/show/get/?%s' % query, show_urlname)['data'][0] + + return self.playlist_result( + self._extract_playlist_entries('http://tv.adobe.com/api/v4/episode/?%s' % query, show_urlname), + str(show_data['id']), + show_data['show_name'], + show_data['show_description']) + + +class AdobeTVChannelIE(AdobeTVPlaylistBaseIE): + _VALID_URL = r'https?://tv\.adobe\.com/(?:(?Pfr|de|es|jp)/)?channel/(?P[^/]+)(?:/(?P[^/]+))?' + + _TEST = { + 'url': 'http://tv.adobe.com/channel/development', + 'info_dict': { + 'id': 'development', + }, + 'playlist_mincount': 96, + } + + def _get_element_url(self, element_data): + return element_data['url'] + + def _real_extract(self, url): + language, channel_urlname, category_urlname = re.match(self._VALID_URL, url).groups() + if not language: + language = 'en' + query = 'language=%s&channel_urlname=%s' % (language, channel_urlname) + if category_urlname: + query += '&category_urlname=%s' % category_urlname + + return self.playlist_result( + self._extract_playlist_entries('http://tv.adobe.com/api/v4/show/?%s' % query, channel_urlname), + channel_urlname) + + class AdobeTVVideoIE(InfoExtractor): _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P\d+)' From 2c3b9f35703def516e455448666af0b4c5b31ec9 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 4 Dec 2015 08:37:08 +0100 Subject: [PATCH 4/5] [adobetv] use a variable for api base url --- youtube_dl/extractor/adobetv.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py index d0bfafa45..d65826932 100644 --- a/youtube_dl/extractor/adobetv.py +++ b/youtube_dl/extractor/adobetv.py @@ -14,7 +14,11 @@ from ..utils import ( ) -class AdobeTVIE(InfoExtractor): +class AdobeTVBaseIE(InfoExtractor): + _API_BASE_URL = 'http://tv.adobe.com/api/v4/' + + +class AdobeTVIE(AdobeTVBaseIE): _VALID_URL = r'https?://tv\.adobe\.com/(?:(?Pfr|de|es|jp)/)?watch/(?P[^/]+)/(?P[^/]+)' _TEST = { @@ -38,7 +42,7 @@ class AdobeTVIE(InfoExtractor): language = 'en' video_data = self._download_json( - 'http://tv.adobe.com/api/v4/episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname), + self._API_BASE_URL + 'episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname), urlname)['data'][0] formats = [{ @@ -62,7 +66,7 @@ class AdobeTVIE(InfoExtractor): } -class AdobeTVPlaylistBaseIE(InfoExtractor): +class AdobeTVPlaylistBaseIE(AdobeTVBaseIE): def _parse_page_data(self, page_data): return [self.url_result(self._get_element_url(element_data)) for element_data in page_data] @@ -97,11 +101,10 @@ class AdobeTVShowIE(AdobeTVPlaylistBaseIE): language = 'en' query = 'language=%s&show_urlname=%s' % (language, show_urlname) - show_data = self._download_json( - 'http://tv.adobe.com/api/v4/show/get/?%s' % query, show_urlname)['data'][0] + show_data = self._download_json(self._API_BASE_URL + 'show/get/?%s' % query, show_urlname)['data'][0] return self.playlist_result( - self._extract_playlist_entries('http://tv.adobe.com/api/v4/episode/?%s' % query, show_urlname), + self._extract_playlist_entries(self._API_BASE_URL + 'episode/?%s' % query, show_urlname), str(show_data['id']), show_data['show_name'], show_data['show_description']) @@ -130,7 +133,7 @@ class AdobeTVChannelIE(AdobeTVPlaylistBaseIE): query += '&category_urlname=%s' % category_urlname return self.playlist_result( - self._extract_playlist_entries('http://tv.adobe.com/api/v4/show/?%s' % query, channel_urlname), + self._extract_playlist_entries(self._API_BASE_URL + 'show/?%s' % query, channel_urlname), channel_urlname) From 7079f8ff1fe524a5711957a426066b94a22a605e Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 4 Dec 2015 08:44:18 +0100 Subject: [PATCH 5/5] [adobetv] use compat_str --- youtube_dl/extractor/adobetv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py index d65826932..8753ee2cf 100644 --- a/youtube_dl/extractor/adobetv.py +++ b/youtube_dl/extractor/adobetv.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..compat import compat_str from ..utils import ( parse_duration, unified_strdate, @@ -55,7 +56,7 @@ class AdobeTVIE(AdobeTVBaseIE): self._sort_formats(formats) return { - 'id': str(video_data['id']), + 'id': compat_str(video_data['id']), 'title': video_data['title'], 'description': video_data.get('description'), 'thumbnail': video_data.get('thumbnail'), @@ -105,7 +106,7 @@ class AdobeTVShowIE(AdobeTVPlaylistBaseIE): return self.playlist_result( self._extract_playlist_entries(self._API_BASE_URL + 'episode/?%s' % query, show_urlname), - str(show_data['id']), + compat_str(show_data['id']), show_data['show_name'], show_data['show_description'])