From fee2d8d9c38f9b5f0a8df347c1e698983339c34d Mon Sep 17 00:00:00 2001 From: gmes78 Date: Thu, 18 Jan 2024 23:41:28 +0000 Subject: [PATCH] [ie/Rule34Video] Extract more metadata (#7416) Closes #7233 Authored by: gmes78 --- yt_dlp/extractor/rule34video.py | 77 +++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/yt_dlp/extractor/rule34video.py b/yt_dlp/extractor/rule34video.py index f3250b557..e6bb4258e 100644 --- a/yt_dlp/extractor/rule34video.py +++ b/yt_dlp/extractor/rule34video.py @@ -1,7 +1,20 @@ import re -from ..utils import parse_duration, unescapeHTML from .common import InfoExtractor +from ..utils import ( + clean_html, + extract_attributes, + get_element_by_attribute, + get_element_by_class, + get_element_html_by_class, + get_elements_by_class, + int_or_none, + join_nonempty, + parse_count, + parse_duration, + unescapeHTML, +) +from ..utils.traversal import traverse_obj class Rule34VideoIE(InfoExtractor): @@ -17,7 +30,16 @@ class Rule34VideoIE(InfoExtractor): 'thumbnail': 'https://rule34video.com/contents/videos_screenshots/3065000/3065157/preview.jpg', 'duration': 347.0, 'age_limit': 18, - 'tags': 'count:14' + 'view_count': int, + 'like_count': int, + 'comment_count': int, + 'timestamp': 1639872000, + 'description': 'https://discord.gg/aBqPrHSHvv', + 'upload_date': '20211219', + 'uploader': 'Sweet HMV', + 'uploader_url': 'https://rule34video.com/members/22119/', + 'categories': ['3D', 'MMD', 'iwara'], + 'tags': 'mincount:10' } }, { @@ -30,7 +52,17 @@ class Rule34VideoIE(InfoExtractor): 'thumbnail': 'https://rule34video.com/contents/videos_screenshots/3065000/3065296/preview.jpg', 'duration': 938.0, 'age_limit': 18, - 'tags': 'count:50' + 'view_count': int, + 'like_count': int, + 'comment_count': int, + 'timestamp': 1640131200, + 'description': '', + 'creator': 'WildeerStudio', + 'upload_date': '20211222', + 'uploader': 'CerZule', + 'uploader_url': 'https://rule34video.com/members/36281/', + 'categories': ['3D', 'Tomb Raider'], + 'tags': 'mincount:40' } }, ] @@ -49,17 +81,44 @@ class Rule34VideoIE(InfoExtractor): 'quality': quality, }) - title = self._html_extract_title(webpage) - thumbnail = self._html_search_regex(r'preview_url:\s+\'([^\']+)\'', webpage, 'thumbnail', default=None) - duration = self._html_search_regex(r'"icon-clock">\s+((?:\d+:?)+)', webpage, 'duration', default=None) + categories, creator, uploader, uploader_url = [None] * 4 + for col in get_elements_by_class('col', webpage): + label = clean_html(get_element_by_class('label', col)) + if label == 'Categories:': + categories = list(map(clean_html, get_elements_by_class('item', col))) + elif label == 'Artist:': + creator = join_nonempty(*map(clean_html, get_elements_by_class('item', col)), delim=', ') + elif label == 'Uploaded By:': + uploader = clean_html(get_element_by_class('name', col)) + uploader_url = extract_attributes(get_element_html_by_class('name', col) or '').get('href') return { + **traverse_obj(self._search_json_ld(webpage, video_id, default={}), ({ + 'title': 'title', + 'view_count': 'view_count', + 'like_count': 'like_count', + 'duration': 'duration', + 'timestamp': 'timestamp', + 'description': 'description', + 'thumbnail': ('thumbnails', 0, 'url'), + })), 'id': video_id, 'formats': formats, - 'title': title, - 'thumbnail': thumbnail, - 'duration': parse_duration(duration), + 'title': self._html_extract_title(webpage), + 'thumbnail': self._html_search_regex( + r'preview_url:\s+\'([^\']+)\'', webpage, 'thumbnail', default=None), + 'duration': parse_duration(self._html_search_regex( + r'"icon-clock">\s+((?:\d+:?)+)', webpage, 'duration', default=None)), + 'view_count': int_or_none(self._html_search_regex( + r'"icon-eye">\s+([ \d]+)', webpage, 'views', default='').replace(' ', '')), + 'like_count': parse_count(get_element_by_class('voters count', webpage)), + 'comment_count': int_or_none(self._search_regex( + r'[^(]+\((\d+)\)', get_element_by_attribute('href', '#tab_comments', webpage), 'comment count', fatal=False)), 'age_limit': 18, + 'creator': creator, + 'uploader': uploader, + 'uploader_url': uploader_url, + 'categories': categories, 'tags': list(map(unescapeHTML, re.findall( r']+\bhref="https://rule34video\.com/tags/\d+/"[^>]*>(?P[^>]*)', webpage))), }