archive
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[facebook] Fix extraction from groups

Browse Source 
Closes #2264, related: #2320
pull/2315/head
pukkandan 3 years ago
parent e9a6a65a55
commit b2db8102dc
No known key found for this signature in database
GPG Key ID: 0F00D95A001F4698
1 changed files with 29 additions and 34 deletions
Show Stats Download Patch File Download Diff File

63
yt_dlp/extractor/facebook.py
Unescape Escape View File

@ -31,6 +31,7 @@ from ..utils import (
url_or_none, url_or_none,
urlencode_postdata, urlencode_postdata,
urljoin, urljoin,
variadic,
) )
@ -164,7 +165,7 @@ class FacebookIE(InfoExtractor):
'info_dict': { 'info_dict': {
'id': '1417995061575415', 'id': '1417995061575415',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Yaroslav Korpan - Довгоочікуване відео', 'title': 'Ukrainian Scientists Worldwide | Довгоочікуване відео',
'description': 'Довгоочікуване відео', 'description': 'Довгоочікуване відео',
'timestamp': 1486648771, 'timestamp': 1486648771,
'upload_date': '20170209', 'upload_date': '20170209',
@ -195,8 +196,8 @@ class FacebookIE(InfoExtractor):
'info_dict': { 'info_dict': {
'id': '202882990186699', 'id': '202882990186699',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Elisabeth Ahtn - Hello? Yes your uber ride is here\n* Jukin...', 'title': 'birb (O v O") | Hello? Yes your uber ride is here',
'description': 'Hello? Yes your uber ride is here\n* Jukin Media Verified *\nFind this video and others like it by visiting...', 'description': 'Hello? Yes your uber ride is here * Jukin Media Verified * Find this video and others like it by visiting...',
'timestamp': 1486035513, 'timestamp': 1486035513,
'upload_date': '20170202', 'upload_date': '20170202',
'uploader': 'Elisabeth Ahtn', 'uploader': 'Elisabeth Ahtn',
@ -398,27 +399,27 @@ class FacebookIE(InfoExtractor):
url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id) url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
def extract_metadata(webpage): def extract_metadata(webpage):
media_data = [self._parse_json(j, video_id, fatal=False) for j in re.findall( post_data = [self._parse_json(j, video_id, fatal=False) for j in re.findall(
r'handleWithCustomApplyEach\(\s*ScheduledApplyEach\s*,\s*(\{.+?\})\s*\);', webpage)] r'handleWithCustomApplyEach\(\s*ScheduledApplyEach\s*,\s*(\{.+?\})\s*\);', webpage)]
media = traverse_obj(media_data, ( post = traverse_obj(post_data, (
..., 'require', ..., ..., ..., '__bbox', 'result', 'data', 'attachments', ..., 'media'), expected_type=dict) ..., 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
media = [m for m in media if str(m.get('id')) == video_id and m.get('__typename') == 'Video'] media = [m for m in traverse_obj(post, (..., 'attachments', ..., 'media'), expected_type=dict) or []
if str(m.get('id')) == video_id and m.get('__typename') == 'Video']
video_title = traverse_obj(media, (..., 'title', 'text'), get_all=False) title = traverse_obj(media, (..., 'title', 'text'), get_all=False)
description = traverse_obj(media, ( description = traverse_obj(media, (
..., 'creation_story', 'comet_sections', 'message', 'story', 'message', 'text'), get_all=False) ..., 'creation_story', 'comet_sections', 'message', 'story', 'message', 'text'), get_all=False)
uploader = traverse_obj(media, (..., 'owner', 'name'), get_all=False) uploader_data = (traverse_obj(media, (..., 'owner'), get_all=False)
uploader_id = traverse_obj(media, (..., 'owner', 'id'), get_all=False) or traverse_obj(post, (..., 'node', 'actors', ...), get_all=False) or {})
video_title = video_title or self._html_search_regex(( page_title = title or self._html_search_regex((
r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>', r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(?P<content>.*?)</span>', r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(?P<content>.*?)</span>',
self._meta_regex('og:title'), self._meta_regex('twitter:title'), self._meta_regex('description'), self._meta_regex('og:title'), self._meta_regex('twitter:title'), r'<title>(?P<content>.+?)</title>'
), webpage, 'title', default=None, group='content') ), webpage, 'title', default=None, group='content')
description = description or self._html_search_meta( description = description or self._html_search_meta(
['description', 'og:description', 'twitter:description'], ['description', 'og:description', 'twitter:description'],
webpage, 'description', default=None) webpage, 'description', default=None)
uploader = uploader or ( uploader = uploader_data.get('name') or (
clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage)) clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
or self._search_regex( or self._search_regex(
(r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes('title')), webpage, 'uploader', fatal=False)) (r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes('title')), webpage, 'uploader', fatal=False))
@ -437,17 +438,17 @@ class FacebookIE(InfoExtractor):
r'\bviewCount\s*:\s*["\']([\d,.]+)', webpage, 'view count', r'\bviewCount\s*:\s*["\']([\d,.]+)', webpage, 'view count',
default=None)) default=None))
info_dict = { info_dict = {
'title': video_title or description.replace('\n', ' ') or f'Facebook video #{video_id}',
'description': description, 'description': description,
'uploader': uploader, 'uploader': uploader,
'uploader_id': uploader_id, 'uploader_id': uploader_data.get('id'),
'timestamp': timestamp, 'timestamp': timestamp,
'thumbnail': thumbnail, 'thumbnail': thumbnail,
'view_count': view_count, 'view_count': view_count,
} }
info_json_ld = self._search_json_ld(webpage, video_id, default={}) info_json_ld = self._search_json_ld(webpage, video_id, default={})
if info_json_ld.get('title'): info_json_ld['title'] = (re.sub(r'\s*\|\s*Facebook$', '', title or info_json_ld.get('title') or page_title or '')
info_json_ld['title'] = re.sub(r'\s*\|\s*Facebook$', '', info_json_ld['title']) or (description or '').replace('\n', ' ') or f'Facebook video #{video_id}')
return merge_dicts(info_json_ld, info_dict) return merge_dicts(info_json_ld, info_dict)
video_data = None video_data = None
@ -554,22 +555,15 @@ class FacebookIE(InfoExtractor):
if media.get('__typename') == 'Video': if media.get('__typename') == 'Video':
return parse_graphql_video(media) return parse_graphql_video(media)
nodes = data.get('nodes') or [] nodes = variadic(traverse_obj(data, 'nodes', 'node') or [])
node = data.get('node') or {} attachments = traverse_obj(nodes, (
if not nodes and node: ..., 'comet_sections', 'content', 'story', (None, 'attached_story'), 'attachments',
nodes.append(node) ..., ('styles', 'style_type_renderer'), 'attachment'), expected_type=dict) or []
for node in nodes: for attachment in attachments:
story = try_get(node, lambda x: x['comet_sections']['content']['story'], dict) or {} ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
attachments = try_get(story, [ for n in ns:
lambda x: x['attached_story']['attachments'], parse_attachment(n)
lambda x: x['attachments'] parse_attachment(attachment)
], list) or []
for attachment in attachments:
attachment = try_get(attachment, lambda x: x['style_type_renderer']['attachment'], dict)
ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
for n in ns:
parse_attachment(n)
parse_attachment(attachment)
edges = try_get(data, lambda x: x['mediaset']['currMedia']['edges'], list) or [] edges = try_get(data, lambda x: x['mediaset']['currMedia']['edges'], list) or []
for edge in edges: for edge in edges:
@ -738,6 +732,7 @@ class FacebookPluginsVideoIE(InfoExtractor):
'info_dict': { 'info_dict': {
'id': '10154383743583686', 'id': '10154383743583686',
'ext': 'mp4', 'ext': 'mp4',
# TODO: Fix title, uploader
'title': 'What to do during the haze?', 'title': 'What to do during the haze?',
'uploader': 'Gov.sg', 'uploader': 'Gov.sg',
'upload_date': '20160826', 'upload_date': '20160826',

Write Preview
Loading…
Cancel
Save