archive
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[extractor/youtube] Misc cleanup

Browse Source 
Authored by: coletdjnz
pull/7279/head
pukkandan 1 year ago
parent c2b801fea5
commit 14a14335b2
No known key found for this signature in database
GPG Key ID: 7EEE9E1E817D0A39
1 changed files with 66 additions and 27 deletions
Show Stats Download Patch File Download Diff File

93
yt_dlp/extractor/youtube.py
Unescape Escape View File

@ -292,6 +292,7 @@ class BadgeType(enum.Enum):
AVAILABILITY_PREMIUM = enum.auto() AVAILABILITY_PREMIUM = enum.auto()
AVAILABILITY_SUBSCRIPTION = enum.auto() AVAILABILITY_SUBSCRIPTION = enum.auto()
LIVE_NOW = enum.auto() LIVE_NOW = enum.auto()
VERIFIED = enum.auto()
class YoutubeBaseInfoExtractor(InfoExtractor): class YoutubeBaseInfoExtractor(InfoExtractor):
@ -791,17 +792,23 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
def _extract_and_report_alerts(self, data, *args, **kwargs): def _extract_and_report_alerts(self, data, *args, **kwargs):
return self._report_alerts(self._extract_alerts(data), *args, **kwargs) return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
def _extract_badges(self, renderer: dict): def _extract_badges(self, badge_list: list):
privacy_icon_map = { """
Extract known BadgeType's from a list of badge renderers.
@returns [{'type': BadgeType}]
"""
icon_type_map = {
'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED, 'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE, 'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC 'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC,
'CHECK_CIRCLE_THICK': BadgeType.VERIFIED,
'OFFICIAL_ARTIST_BADGE': BadgeType.VERIFIED,
} }
badge_style_map = { badge_style_map = {
'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION, 'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM, 'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW 'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
} }
label_map = { label_map = {
@ -809,13 +816,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
'private': BadgeType.AVAILABILITY_PRIVATE, 'private': BadgeType.AVAILABILITY_PRIVATE,
'members only': BadgeType.AVAILABILITY_SUBSCRIPTION, 'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
'live': BadgeType.LIVE_NOW, 'live': BadgeType.LIVE_NOW,
'premium': BadgeType.AVAILABILITY_PREMIUM 'premium': BadgeType.AVAILABILITY_PREMIUM,
} }
badges = [] badges = []
for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer')): for badge in traverse_obj(badge_list, (..., lambda key, _: re.search(r'[bB]adgeRenderer$', key))):
badge_type = ( badge_type = (
privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str)) icon_type_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
or badge_style_map.get(traverse_obj(badge, 'style')) or badge_style_map.get(traverse_obj(badge, 'style'))
) )
if badge_type: if badge_type:
@ -823,11 +830,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
continue continue
# fallback, won't work in some languages # fallback, won't work in some languages
label = traverse_obj(badge, 'label', expected_type=str, default='') label = traverse_obj(
badge, 'label', ('accessibilityData', 'label'), 'tooltip', 'iconTooltip', get_all=False, expected_type=str, default='')
for match, label_badge_type in label_map.items(): for match, label_badge_type in label_map.items():
if match in label.lower(): if match in label.lower():
badges.append({'type': badge_type}) badges.append({'type': label_badge_type})
continue break
return badges return badges
@ -1020,8 +1028,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
overlay_style = traverse_obj( overlay_style = traverse_obj(
renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'), renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
get_all=False, expected_type=str) get_all=False, expected_type=str)
badges = self._extract_badges(renderer) badges = self._extract_badges(traverse_obj(renderer, 'badges'))
navigation_url = urljoin('https://www.youtube.com/', traverse_obj( navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'), renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
expected_type=str)) or '' expected_type=str)) or ''
@ -1079,7 +1086,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None, needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None), is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
view_count_field: view_count, view_count_field: view_count,
'live_status': live_status 'live_status': live_status,
} }
@ -1332,6 +1339,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Philipp Hagemeister', 'uploader': 'Philipp Hagemeister',
'uploader_url': 'https://www.youtube.com/@PhilippHagemeister', 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
'uploader_id': '@PhilippHagemeister', 'uploader_id': '@PhilippHagemeister',
'heatmap': 'count:100',
}, },
'params': { 'params': {
'skip_download': True, 'skip_download': True,
@ -1415,6 +1423,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'The Witcher', 'uploader': 'The Witcher',
'uploader_url': 'https://www.youtube.com/@thewitcher', 'uploader_url': 'https://www.youtube.com/@thewitcher',
'uploader_id': '@thewitcher', 'uploader_id': '@thewitcher',
'comment_count': int,
'heatmap': 'count:100',
}, },
}, },
{ {
@ -1894,6 +1904,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Bernie Sanders', 'uploader': 'Bernie Sanders',
'uploader_url': 'https://www.youtube.com/@BernieSanders', 'uploader_url': 'https://www.youtube.com/@BernieSanders',
'uploader_id': '@BernieSanders', 'uploader_id': '@BernieSanders',
'heatmap': 'count:100',
}, },
'params': { 'params': {
'skip_download': True, 'skip_download': True,
@ -1955,6 +1966,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Vsauce', 'uploader': 'Vsauce',
'uploader_url': 'https://www.youtube.com/@Vsauce', 'uploader_url': 'https://www.youtube.com/@Vsauce',
'uploader_id': '@Vsauce', 'uploader_id': '@Vsauce',
'comment_count': int,
}, },
'params': { 'params': {
'skip_download': True, 'skip_download': True,
@ -2147,6 +2159,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'kudvenkat', 'uploader': 'kudvenkat',
'uploader_url': 'https://www.youtube.com/@Csharp-video-tutorialsBlogspot', 'uploader_url': 'https://www.youtube.com/@Csharp-video-tutorialsBlogspot',
'uploader_id': '@Csharp-video-tutorialsBlogspot', 'uploader_id': '@Csharp-video-tutorialsBlogspot',
'heatmap': 'count:100',
}, },
'params': { 'params': {
'skip_download': True, 'skip_download': True,
@ -2227,6 +2240,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'CBS Mornings', 'uploader': 'CBS Mornings',
'uploader_url': 'https://www.youtube.com/@CBSMornings', 'uploader_url': 'https://www.youtube.com/@CBSMornings',
'uploader_id': '@CBSMornings', 'uploader_id': '@CBSMornings',
'comment_count': int,
} }
}, },
{ {
@ -2297,6 +2311,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'colinfurze', 'uploader': 'colinfurze',
'uploader_url': 'https://www.youtube.com/@colinfurze', 'uploader_url': 'https://www.youtube.com/@colinfurze',
'uploader_id': '@colinfurze', 'uploader_id': '@colinfurze',
'comment_count': int,
'heatmap': 'count:100',
}, },
'params': { 'params': {
'format': '17', # 3gp format available on android 'format': '17', # 3gp format available on android
@ -2342,6 +2358,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'SciShow', 'uploader': 'SciShow',
'uploader_url': 'https://www.youtube.com/@SciShow', 'uploader_url': 'https://www.youtube.com/@SciShow',
'uploader_id': '@SciShow', 'uploader_id': '@SciShow',
'comment_count': int,
'heatmap': 'count:100',
}, 'params': {'format': 'mhtml', 'skip_download': True} }, 'params': {'format': 'mhtml', 'skip_download': True}
}, { }, {
# Ensure video upload_date is in UTC timezone (video was uploaded 1641170939) # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
@ -2370,6 +2388,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Leon Nguyen', 'uploader': 'Leon Nguyen',
'uploader_url': 'https://www.youtube.com/@LeonNguyen', 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
'uploader_id': '@LeonNguyen', 'uploader_id': '@LeonNguyen',
'heatmap': 'count:100',
} }
}, { }, {
# Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
@ -2398,6 +2417,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Leon Nguyen', 'uploader': 'Leon Nguyen',
'uploader_url': 'https://www.youtube.com/@LeonNguyen', 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
'uploader_id': '@LeonNguyen', 'uploader_id': '@LeonNguyen',
'heatmap': 'count:100',
}, },
'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']} 'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
}, { }, {
@ -2428,6 +2448,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Quackity', 'uploader': 'Quackity',
'uploader_id': '@Quackity', 'uploader_id': '@Quackity',
'uploader_url': 'https://www.youtube.com/@Quackity', 'uploader_url': 'https://www.youtube.com/@Quackity',
'comment_count': int,
'heatmap': 'count:100',
} }
}, },
{ # continuous livestream. Microformat upload date should be preferred. { # continuous livestream. Microformat upload date should be preferred.
@ -2594,6 +2616,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'MrBeast', 'uploader': 'MrBeast',
'uploader_url': 'https://www.youtube.com/@MrBeast', 'uploader_url': 'https://www.youtube.com/@MrBeast',
'uploader_id': '@MrBeast', 'uploader_id': '@MrBeast',
'comment_count': int,
'heatmap': 'count:100',
}, },
'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'}, 'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
}, { }, {
@ -2655,6 +2679,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'さなちゃんねる', 'uploader': 'さなちゃんねる',
'uploader_url': 'https://www.youtube.com/@sana_natori', 'uploader_url': 'https://www.youtube.com/@sana_natori',
'uploader_id': '@sana_natori', 'uploader_id': '@sana_natori',
'heatmap': 'count:100',
}, },
}, },
{ {
@ -2684,6 +2709,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'thumbnail': r're:^https?://.*\.webp', 'thumbnail': r're:^https?://.*\.webp',
'channel_url': 'https://www.youtube.com/channel/UCxzC4EngIsMrPmbm6Nxvb-A', 'channel_url': 'https://www.youtube.com/channel/UCxzC4EngIsMrPmbm6Nxvb-A',
'playable_in_embed': True, 'playable_in_embed': True,
'comment_count': int,
'heatmap': 'count:100',
}, },
'params': { 'params': {
'extractor_args': {'youtube': {'player_client': ['android'], 'player_skip': ['webpage']}}, 'extractor_args': {'youtube': {'player_client': ['android'], 'player_skip': ['webpage']}},
@ -2720,6 +2747,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Christopher Sykes', 'uploader': 'Christopher Sykes',
'uploader_url': 'https://www.youtube.com/@ChristopherSykesDocumentaries', 'uploader_url': 'https://www.youtube.com/@ChristopherSykesDocumentaries',
'uploader_id': '@ChristopherSykesDocumentaries', 'uploader_id': '@ChristopherSykesDocumentaries',
'heatmap': 'count:100',
}, },
'params': { 'params': {
'skip_download': True, 'skip_download': True,
@ -3312,10 +3340,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if comment_abr is not None: if comment_abr is not None:
info['is_favorited'] = 'creatorHeart' in comment_abr info['is_favorited'] = 'creatorHeart' in comment_abr
comment_ab_icontype = traverse_obj( badges = self._extract_badges([traverse_obj(comment_renderer, 'authorCommentBadge')])
comment_renderer, ('authorCommentBadge', 'authorCommentBadgeRenderer', 'icon', 'iconType')) if self._has_badge(badges, BadgeType.VERIFIED):
if comment_ab_icontype is not None: info['author_is_verified'] = True
info['author_is_verified'] = comment_ab_icontype in ('CHECK_CIRCLE_THICK', 'OFFICIAL_ARTIST_BADGE')
is_pinned = traverse_obj(comment_renderer, 'pinnedCommentBadge') is_pinned = traverse_obj(comment_renderer, 'pinnedCommentBadge')
if is_pinned: if is_pinned:
@ -4481,7 +4508,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if v: if v:
info[d_k] = v info[d_k] = v
badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False)) badges = self._extract_badges(traverse_obj(vpir, 'badges'))
is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
or get_first(video_details, 'isPrivate', expected_type=bool)) or get_first(video_details, 'isPrivate', expected_type=bool))
@ -4554,13 +4581,14 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
channel_id = self.ucid_or_none(renderer['channelId']) channel_id = self.ucid_or_none(renderer['channelId'])
title = self._get_text(renderer, 'title') title = self._get_text(renderer, 'title')
channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None) channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None)
# As of 2023-03-01 YouTube doesn't use the channel handles on these renderers yet.
# However we can expect them to change that in the future.
channel_handle = self.handle_from_url( channel_handle = self.handle_from_url(
traverse_obj(renderer, ( traverse_obj(renderer, (
'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'), 'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'),
('browseEndpoint', 'canonicalBaseUrl')), ('browseEndpoint', 'canonicalBaseUrl')),
{str}), get_all=False)) {str}), get_all=False))
if not channel_handle:
# As of 2023-06-01, YouTube sets subscriberCountText to the handle in search
channel_handle = self.handle_or_none(self._get_text(renderer, 'subscriberCountText'))
return { return {
'_type': 'url', '_type': 'url',
'url': channel_url, 'url': channel_url,
@ -4573,9 +4601,15 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
'title': title, 'title': title,
'uploader_id': channel_handle, 'uploader_id': channel_handle,
'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None), 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
'channel_follower_count': self._get_count(renderer, 'subscriberCountText'), # See above. YouTube sets videoCountText to the subscriber text in search channel renderers.
# However, in feed/channels this is set correctly to the subscriber count
'channel_follower_count': traverse_obj(
renderer, 'subscriberCountText', 'videoCountText', expected_type=self._get_count),
'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'), 'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
'playlist_count': self._get_count(renderer, 'videoCountText'), 'playlist_count': (
# videoCountText may be the subscriber count
self._get_count(renderer, 'videoCountText')
if self._get_count(renderer, 'subscriberCountText') is not None else None),
'description': self._get_text(renderer, 'descriptionSnippet'), 'description': self._get_text(renderer, 'descriptionSnippet'),
} }
@ -5100,7 +5134,7 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {} playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
player_header_privacy = playlist_header_renderer.get('privacy') player_header_privacy = playlist_header_renderer.get('privacy')
badges = self._extract_badges(sidebar_renderer) badges = self._extract_badges(traverse_obj(sidebar_renderer, 'badges'))
# Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
privacy_setting_icon = get_first( privacy_setting_icon = get_first(
@ -5350,7 +5384,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'uploader_url': 'https://www.youtube.com/@3blue1brown', 'uploader_url': 'https://www.youtube.com/@3blue1brown',
'uploader': '3Blue1Brown', 'uploader': '3Blue1Brown',
'tags': ['Mathematics'], 'tags': ['Mathematics'],
'channel_follower_count': int 'channel_follower_count': int,
}, },
}, { }, {
'note': 'playlists, singlepage', 'note': 'playlists, singlepage',
@ -5690,7 +5724,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
}, { }, {
'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live', 'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
'info_dict': { 'info_dict': {
'id': 'AlTsmyW4auo', # This will keep changing 'id': 'hGkQjiJLjWQ', # This will keep changing
'ext': 'mp4', 'ext': 'mp4',
'title': str, 'title': str,
'upload_date': r're:\d{8}', 'upload_date': r're:\d{8}',
@ -6202,7 +6236,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'channel': str, 'channel': str,
'uploader': str, 'uploader': str,
'uploader_url': str, 'uploader_url': str,
'uploader_id': str 'uploader_id': str,
} }
}], }],
'params': {'extract_flat': True, 'playlist_items': '1'}, 'params': {'extract_flat': True, 'playlist_items': '1'},
@ -6865,12 +6899,14 @@ class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc', 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
'title': 'Kurzgesagt – In a Nutshell', 'title': 'Kurzgesagt – In a Nutshell',
'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q', 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
'playlist_count': int, # XXX: should have a way of saying > 1 # No longer available for search as it is set to the handle.
# 'playlist_count': int,
'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q', 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
'thumbnails': list, 'thumbnails': list,
'uploader_id': '@kurzgesagt', 'uploader_id': '@kurzgesagt',
'uploader_url': 'https://www.youtube.com/@kurzgesagt', 'uploader_url': 'https://www.youtube.com/@kurzgesagt',
'uploader': 'Kurzgesagt – In a Nutshell', 'uploader': 'Kurzgesagt – In a Nutshell',
'channel_follower_count': int,
} }
}], }],
'params': {'extract_flat': True, 'playlist_items': '1'}, 'params': {'extract_flat': True, 'playlist_items': '1'},
@ -7134,6 +7170,8 @@ class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
'live_status': 'not_live', 'live_status': 'not_live',
'channel_follower_count': int, 'channel_follower_count': int,
'chapters': 'count:20', 'chapters': 'count:20',
'comment_count': int,
'heatmap': 'count:100',
} }
}] }]
@ -7194,6 +7232,7 @@ class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
'channel': 'さなちゃんねる', 'channel': 'さなちゃんねる',
'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d', 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
'uploader': 'さなちゃんねる', 'uploader': 'さなちゃんねる',
'heatmap': 'count:100',
}, },
'add_ie': ['Youtube'], 'add_ie': ['Youtube'],
'params': {'skip_download': 'Youtube'}, 'params': {'skip_download': 'Youtube'},

Write Preview
Loading…
Cancel
Save