From 993edd3f6e17e966c763bc86dc34125445cec6b6 Mon Sep 17 00:00:00 2001
From: pukkandan
Date: Wed, 6 Dec 2023 03:44:11 +0530
Subject: [PATCH 001/248] [outtmpl] Support multiplication
Related: #8683
---
README.md | 2 +-
test/test_YoutubeDL.py | 1 +
yt_dlp/YoutubeDL.py | 1 +
3 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index f67cab572..78d4799a1 100644
--- a/README.md
+++ b/README.md
@@ -1268,7 +1268,7 @@ The field names themselves (the part inside the parenthesis) can also have some
1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a dot `.` separator; e.g. `%(tags.0)s`, `%(subtitles.en.-1.ext)s`. You can do Python slicing with colon `:`; E.g. `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. Curly braces `{}` can be used to build dictionaries with only specific keys; e.g. `%(formats.:.{format_id,height})#j`. An empty field name `%()s` refers to the entire infodict; e.g. `%(.{id,title})s`. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
-1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. E.g. `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
+1. **Arithmetic**: Simple arithmetic can be done on numeric fields using `+`, `-` and `*`. E.g. `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. E.g. `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 0cf130db0..48c710e00 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -797,6 +797,7 @@ class TestYoutubeDL(unittest.TestCase):
test('%(title|%)s %(title|%%)s', '% %%')
test('%(id+1-height+3)05d', '00158')
test('%(width+100)05d', 'NA')
+ test('%(filesize*8)d', '8192')
test('%(formats.0) 15s', ('% 15s' % FORMATS[0], None))
test('%(formats.0)r', (repr(FORMATS[0]), None))
test('%(height.0)03d', '001')
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index e65bef862..29dd76186 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1179,6 +1179,7 @@ class YoutubeDL:
MATH_FUNCTIONS = {
'+': float.__add__,
'-': float.__sub__,
+ '*': float.__mul__,
}
# Field is of the form key1.key2...
# where keys (except first) can be string, int, slice or "{field, ...}"
From 044886c220620a7679109e92352890e18b6079e3 Mon Sep 17 00:00:00 2001
From: pukkandan
Date: Wed, 6 Dec 2023 03:31:45 +0530
Subject: [PATCH 002/248] [ie/youtube] Return empty playlist when channel/tab
has no videos
Closes #8634
---
yt_dlp/extractor/youtube.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index b6212646e..86818a9dc 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -6469,6 +6469,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
def _has_tab(self, tabs, tab_id):
return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
+ def _empty_playlist(self, item_id, data):
+ return self.playlist_result([], item_id, **self._extract_metadata_from_tabs(item_id, data))
+
@YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
def _real_extract(self, url, smuggled_data):
item_id = self._match_id(url)
@@ -6534,6 +6537,10 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url) # NB: Name may be translated
self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
+ # /about is no longer a tab
+ if original_tab_id == 'about':
+ return self._empty_playlist(item_id, data)
+
if not original_tab_id and selected_tab_name:
self.to_screen('Downloading all uploads of the channel. '
'To download only the videos in a specific tab, pass the tab\'s URL')
@@ -6546,7 +6553,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
if not extra_tabs and selected_tab_id != 'videos':
# Channel does not have streams, shorts or videos tabs
if item_id[:2] != 'UC':
- raise ExtractorError('This channel has no uploads', expected=True)
+ return self._empty_playlist(item_id, data)
# Topic channels don't have /videos. Use the equivalent playlist instead
pl_id = f'UU{item_id[2:]}'
@@ -6554,7 +6561,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
try:
data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
except ExtractorError:
- raise ExtractorError('This channel has no uploads', expected=True)
+ return self._empty_playlist(item_id, data)
else:
item_id, url = pl_id, pl_url
self.to_screen(
From 71f28097fec1c9e029f74b68a4eadc8915399840 Mon Sep 17 00:00:00 2001
From: Pierrick Guillaume <34305318+Fymyte@users.noreply.github.com>
Date: Wed, 6 Dec 2023 16:10:11 +0100
Subject: [PATCH 003/248] [ie/francetv] Improve metadata extraction (#8409)
Authored by: Fymyte
---
yt_dlp/extractor/francetv.py | 41 ++++++++++++++++++++++++++++--------
1 file changed, 32 insertions(+), 9 deletions(-)
diff --git a/yt_dlp/extractor/francetv.py b/yt_dlp/extractor/francetv.py
index 052317204..0ceecde74 100644
--- a/yt_dlp/extractor/francetv.py
+++ b/yt_dlp/extractor/francetv.py
@@ -1,12 +1,14 @@
from .common import InfoExtractor
+from .dailymotion import DailymotionIE
from ..utils import (
- determine_ext,
ExtractorError,
+ determine_ext,
format_field,
+ int_or_none,
+ join_nonempty,
parse_iso8601,
parse_qs,
)
-from .dailymotion import DailymotionIE
class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -82,6 +84,8 @@ class FranceTVIE(InfoExtractor):
videos = []
title = None
subtitle = None
+ episode_number = None
+ season_number = None
image = None
duration = None
timestamp = None
@@ -112,7 +116,9 @@ class FranceTVIE(InfoExtractor):
if meta:
if title is None:
title = meta.get('title')
- # XXX: what is meta['pre_title']?
+ # meta['pre_title'] contains season and episode number for series in format "S E"
+ season_number, episode_number = self._search_regex(
+ r'S(\d+)\s*E(\d+)', meta.get('pre_title'), 'episode info', group=(1, 2), default=(None, None))
if subtitle is None:
subtitle = meta.get('additional_title')
if image is None:
@@ -191,19 +197,19 @@ class FranceTVIE(InfoExtractor):
} for sheet in spritesheets]
})
- if subtitle:
- title += ' - %s' % subtitle
- title = title.strip()
-
return {
'id': video_id,
- 'title': title,
+ 'title': join_nonempty(title, subtitle, delim=' - ').strip(),
'thumbnail': image,
'duration': duration,
'timestamp': timestamp,
'is_live': is_live,
'formats': formats,
'subtitles': subtitles,
+ 'episode': subtitle if episode_number else None,
+ 'series': title if episode_number else None,
+ 'episode_number': int_or_none(episode_number),
+ 'season_number': int_or_none(season_number),
}
def _real_extract(self, url):
@@ -230,14 +236,31 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor):
'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1',
'ext': 'mp4',
'title': '13h15, le dimanche... - Les mystères de Jésus',
- 'description': 'md5:75efe8d4c0a8205e5904498ffe1e1a42',
'timestamp': 1502623500,
+ 'duration': 2580,
+ 'thumbnail': r're:^https?://.*\.jpg$',
'upload_date': '20170813',
},
'params': {
'skip_download': True,
},
'add_ie': [FranceTVIE.ie_key()],
+ }, {
+ 'url': 'https://www.france.tv/enfants/six-huit-ans/foot2rue/saison-1/3066387-duel-au-vieux-port.html',
+ 'info_dict': {
+ 'id': 'a9050959-eedd-4b4a-9b0d-de6eeaa73e44',
+ 'ext': 'mp4',
+ 'title': 'Foot2Rue - Duel au vieux port',
+ 'episode': 'Duel au vieux port',
+ 'series': 'Foot2Rue',
+ 'episode_number': 1,
+ 'season_number': 1,
+ 'timestamp': 1642761360,
+ 'upload_date': '20220121',
+ 'season': 'Season 1',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'duration': 1441,
+ },
}, {
# france3
'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html',
From b03c89309eb141be1a1eceeeb7475dd3b7529ad9 Mon Sep 17 00:00:00 2001
From: Nicolas Cisco
Date: Wed, 6 Dec 2023 14:55:38 -0300
Subject: [PATCH 004/248] [ie/mediastream] Fix authenticated format extraction
(#8657)
Authored by: NickCis
---
yt_dlp/extractor/mediastream.py | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py
index b8cb5a691..ae0fb2aed 100644
--- a/yt_dlp/extractor/mediastream.py
+++ b/yt_dlp/extractor/mediastream.py
@@ -3,8 +3,11 @@ import re
from .common import InfoExtractor
from ..utils import (
clean_html,
+ filter_dict,
+ parse_qs,
remove_end,
traverse_obj,
+ update_url_query,
urljoin,
)
@@ -108,7 +111,9 @@ class MediaStreamIE(MediaStreamBaseIE):
for message in [
'Debido a tu ubicación no puedes ver el contenido',
- 'You are not allowed to watch this video: Geo Fencing Restriction'
+ 'You are not allowed to watch this video: Geo Fencing Restriction',
+ 'Este contenido no está disponible en tu zona geográfica.',
+ 'El contenido sólo está disponible dentro de',
]:
if message in webpage:
self.raise_geo_restricted()
@@ -118,7 +123,16 @@ class MediaStreamIE(MediaStreamBaseIE):
formats, subtitles = [], {}
for video_format in player_config['src']:
if video_format == 'hls':
- fmts, subs = self._extract_m3u8_formats_and_subtitles(player_config['src'][video_format], video_id)
+ params = {
+ 'at': 'web-app',
+ 'access_token': traverse_obj(parse_qs(url), ('access_token', 0)),
+ }
+ for name, key in (('MDSTRMUID', 'uid'), ('MDSTRMSID', 'sid'), ('MDSTRMPID', 'pid'), ('VERSION', 'av')):
+ params[key] = self._search_regex(
+ rf'window\.{name}\s*=\s*["\']([^"\']+)["\'];', webpage, key, default=None)
+
+ fmts, subs = self._extract_m3u8_formats_and_subtitles(
+ update_url_query(player_config['src'][video_format], filter_dict(params)), video_id)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
elif video_format == 'mpd':
From 04a5e06350e3ef7c03f94f2f3f90dd96c6411152 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Wed, 6 Dec 2023 18:58:00 +0100
Subject: [PATCH 005/248] [ie/ondemandkorea] Fix upgraded format extraction
(#8677)
Closes #8675
Authored by: seproDev
---
yt_dlp/extractor/ondemandkorea.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/yt_dlp/extractor/ondemandkorea.py b/yt_dlp/extractor/ondemandkorea.py
index 81ce99fd9..94fcac720 100644
--- a/yt_dlp/extractor/ondemandkorea.py
+++ b/yt_dlp/extractor/ondemandkorea.py
@@ -3,7 +3,6 @@ import re
import uuid
from .common import InfoExtractor
-from ..networking import HEADRequest
from ..utils import (
ExtractorError,
OnDemandPagedList,
@@ -84,15 +83,17 @@ class OnDemandKoreaIE(InfoExtractor):
def try_geo_bypass(url):
return traverse_obj(url, ({parse_qs}, 'stream_url', 0, {url_or_none})) or url
- def try_upgrade_quality(url):
- mod_url = re.sub(r'_720(p?)\.m3u8', r'_1080\1.m3u8', url)
- return mod_url if mod_url != url and self._request_webpage(
- HEADRequest(mod_url), video_id, note='Checking for higher quality format',
- errnote='No higher quality format found', fatal=False) else url
-
formats = []
for m3u8_url in traverse_obj(data, (('sources', 'manifest'), ..., 'url', {url_or_none}, {try_geo_bypass})):
- formats.extend(self._extract_m3u8_formats(try_upgrade_quality(m3u8_url), video_id, fatal=False))
+ mod_url = re.sub(r'_720(p?)\.m3u8', r'_1080\1.m3u8', m3u8_url)
+ if mod_url != m3u8_url:
+ mod_format = self._extract_m3u8_formats(
+ mod_url, video_id, note='Checking for higher quality format',
+ errnote='No higher quality format found', fatal=False)
+ if mod_format:
+ formats.extend(mod_format)
+ continue
+ formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, fatal=False))
subtitles = {}
for track in traverse_obj(data, ('text_tracks', lambda _, v: url_or_none(v['url']))):
From f98a3305eb124a0c375d03209d5c5a64fe1766c8 Mon Sep 17 00:00:00 2001
From: Simon Sawicki
Date: Wed, 6 Dec 2023 21:44:54 +0100
Subject: [PATCH 006/248] [ie/pr0gramm] Support variant formats and subtitles
(#8674)
Authored by: Grub4K
---
yt_dlp/extractor/pr0gramm.py | 56 +++++++++++++++++++++++++++++-------
1 file changed, 45 insertions(+), 11 deletions(-)
diff --git a/yt_dlp/extractor/pr0gramm.py b/yt_dlp/extractor/pr0gramm.py
index c8e0bb493..2a6794208 100644
--- a/yt_dlp/extractor/pr0gramm.py
+++ b/yt_dlp/extractor/pr0gramm.py
@@ -4,7 +4,14 @@ from urllib.parse import unquote
from .common import InfoExtractor
from ..compat import functools
-from ..utils import ExtractorError, make_archive_id, urljoin
+from ..utils import (
+ ExtractorError,
+ float_or_none,
+ int_or_none,
+ make_archive_id,
+ mimetype2ext,
+ urljoin,
+)
from ..utils.traversal import traverse_obj
@@ -26,6 +33,7 @@ class Pr0grammIE(InfoExtractor):
'dislike_count': int,
'age_limit': 0,
'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+ '_old_archive_ids': ['pr0grammstatic 5466437'],
},
}, {
# Tags require account
@@ -43,6 +51,7 @@ class Pr0grammIE(InfoExtractor):
'dislike_count': int,
'age_limit': 0,
'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+ '_old_archive_ids': ['pr0grammstatic 3052805'],
},
}, {
# Requires verified account
@@ -60,6 +69,7 @@ class Pr0grammIE(InfoExtractor):
'dislike_count': int,
'age_limit': 18,
'thumbnail': r're:^https://thumb\.pr0gramm\.com/.*\.jpg',
+ '_old_archive_ids': ['pr0grammstatic 5848332'],
},
}, {
'url': 'https://pr0gramm.com/static/5466437',
@@ -110,37 +120,61 @@ class Pr0grammIE(InfoExtractor):
return data
+ @staticmethod
+ def _create_source_url(path):
+ return urljoin('https://img.pr0gramm.com', path)
+
def _real_extract(self, url):
video_id = self._match_id(url)
video_info = traverse_obj(
self._call_api('get', video_id, {'id': video_id, 'flags': self._maximum_flags}),
('items', 0, {dict}))
- source = urljoin('https://img.pr0gramm.com', video_info.get('image'))
+ source = video_info.get('image')
if not source or not source.endswith('mp4'):
self.raise_no_formats('Could not extract a video', expected=bool(source), video_id=video_id)
tags = None
if self._is_logged_in:
- metadata = self._call_api('info', video_id, {'itemId': video_id})
+ metadata = self._call_api('info', video_id, {'itemId': video_id}, note='Downloading tags')
tags = traverse_obj(metadata, ('tags', ..., 'tag', {str}))
# Sorted by "confidence", higher confidence = earlier in list
confidences = traverse_obj(metadata, ('tags', ..., 'confidence', ({int}, {float})))
if confidences:
tags = [tag for _, tag in sorted(zip(confidences, tags), reverse=True)]
+ formats = traverse_obj(video_info, ('variants', ..., {
+ 'format_id': ('name', {str}),
+ 'url': ('path', {self._create_source_url}),
+ 'ext': ('mimeType', {mimetype2ext}),
+ 'vcodec': ('codec', {str}),
+ 'width': ('width', {int_or_none}),
+ 'height': ('height', {int_or_none}),
+ 'bitrate': ('bitRate', {float_or_none}),
+ 'filesize': ('fileSize', {int_or_none}),
+ })) if video_info.get('variants') else [{
+ 'ext': 'mp4',
+ 'format_id': 'source',
+ **traverse_obj(video_info, {
+ 'url': ('image', {self._create_source_url}),
+ 'width': ('width', {int_or_none}),
+ 'height': ('height', {int_or_none}),
+ }),
+ }]
+
+ subtitles = {}
+ for subtitle in traverse_obj(video_info, ('subtitles', lambda _, v: v['language'])):
+ subtitles.setdefault(subtitle['language'], []).append(traverse_obj(subtitle, {
+ 'url': ('path', {self._create_source_url}),
+ 'note': ('label', {str}),
+ }))
+
return {
'id': video_id,
'title': f'pr0gramm-{video_id} by {video_info.get("user")}',
- 'formats': [{
- 'url': source,
- 'ext': 'mp4',
- **traverse_obj(video_info, {
- 'width': ('width', {int}),
- 'height': ('height', {int}),
- }),
- }],
'tags': tags,
+ 'formats': formats,
+ 'subtitles': subtitles,
'age_limit': 18 if traverse_obj(video_info, ('flags', {0b110.__and__})) else 0,
'_old_archive_ids': [make_archive_id('Pr0grammStatic', video_id)],
**traverse_obj(video_info, {
From 0b6f829b1dfda15d3c1d7d1fbe4ea6102c26dd24 Mon Sep 17 00:00:00 2001
From: Simon Sawicki
Date: Wed, 6 Dec 2023 21:46:45 +0100
Subject: [PATCH 007/248] [utils] `traverse_obj`: Move `is_user_input` into
output template (#8673)
Authored by: Grub4K
---
test/test_utils.py | 17 -----------------
yt_dlp/YoutubeDL.py | 14 ++++++++++++--
yt_dlp/utils/traversal.py | 19 ++++++-------------
3 files changed, 18 insertions(+), 32 deletions(-)
diff --git a/test/test_utils.py b/test/test_utils.py
index 77040f29c..100f11788 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2317,23 +2317,6 @@ Line 1
self.assertEqual(traverse_obj({}, (0, slice(1)), traverse_string=True), [],
msg='branching should result in list if `traverse_string`')
- # Test is_user_input behavior
- _IS_USER_INPUT_DATA = {'range8': list(range(8))}
- self.assertEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3'),
- is_user_input=True), 3,
- msg='allow for string indexing if `is_user_input`')
- self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3:'),
- is_user_input=True), tuple(range(8))[3:],
- msg='allow for string slice if `is_user_input`')
- self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':4:2'),
- is_user_input=True), tuple(range(8))[:4:2],
- msg='allow step in string slice if `is_user_input`')
- self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':'),
- is_user_input=True), range(8),
- msg='`:` should be treated as `...` if `is_user_input`')
- with self.assertRaises(TypeError, msg='too many params should result in error'):
- traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':::'), is_user_input=True)
-
# Test re.Match as input obj
mobj = re.fullmatch(r'0(12)(?P3)(4)?', '0123')
self.assertEqual(traverse_obj(mobj, ...), [x for x in mobj.groups() if x is not None],
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 29dd76186..0c07866e4 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -1201,6 +1201,15 @@ class YoutubeDL:
(?:\|(?P.*?))?
)$''')
+ def _from_user_input(field):
+ if field == ':':
+ return ...
+ elif ':' in field:
+ return slice(*map(int_or_none, field.split(':')))
+ elif int_or_none(field) is not None:
+ return int(field)
+ return field
+
def _traverse_infodict(fields):
fields = [f for x in re.split(r'\.({.+?})\.?', fields)
for f in ([x] if x.startswith('{') else x.split('.'))]
@@ -1210,11 +1219,12 @@ class YoutubeDL:
for i, f in enumerate(fields):
if not f.startswith('{'):
+ fields[i] = _from_user_input(f)
continue
assert f.endswith('}'), f'No closing brace for {f} in {fields}'
- fields[i] = {k: k.split('.') for k in f[1:-1].split(',')}
+ fields[i] = {k: list(map(_from_user_input, k.split('.'))) for k in f[1:-1].split(',')}
- return traverse_obj(info_dict, fields, is_user_input=True, traverse_string=True)
+ return traverse_obj(info_dict, fields, traverse_string=True)
def get_value(mdict):
# Object traversal
diff --git a/yt_dlp/utils/traversal.py b/yt_dlp/utils/traversal.py
index 462c3ba5d..ff5703198 100644
--- a/yt_dlp/utils/traversal.py
+++ b/yt_dlp/utils/traversal.py
@@ -8,7 +8,7 @@ from ._utils import (
IDENTITY,
NO_DEFAULT,
LazyList,
- int_or_none,
+ deprecation_warning,
is_iterable_like,
try_call,
variadic,
@@ -17,7 +17,7 @@ from ._utils import (
def traverse_obj(
obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
- casesense=True, is_user_input=False, traverse_string=False):
+ casesense=True, is_user_input=NO_DEFAULT, traverse_string=False):
"""
Safely traverse nested `dict`s and `Iterable`s
@@ -63,10 +63,8 @@ def traverse_obj(
@param get_all If `False`, return the first matching result, otherwise all matching ones.
@param casesense If `False`, consider string dictionary keys as case insensitive.
- The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
+ `traverse_string` is only meant to be used by YoutubeDL.prepare_outtmpl and is not part of the API
- @param is_user_input Whether the keys are generated from user input.
- If `True` strings get converted to `int`/`slice` if needed.
@param traverse_string Whether to traverse into objects as strings.
If `True`, any non-compatible object will first be
converted into a string and then traversed into.
@@ -80,6 +78,9 @@ def traverse_obj(
If no `default` is given and the last path branches, a `list` of results
is always returned. If a path ends on a `dict` that result will always be a `dict`.
"""
+ if is_user_input is not NO_DEFAULT:
+ deprecation_warning('The is_user_input parameter is deprecated and no longer works')
+
casefold = lambda k: k.casefold() if isinstance(k, str) else k
if isinstance(expected_type, type):
@@ -195,14 +196,6 @@ def traverse_obj(
key = None
for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
- if is_user_input and isinstance(key, str):
- if key == ':':
- key = ...
- elif ':' in key:
- key = slice(*map(int_or_none, key.split(':')))
- elif int_or_none(key) is not None:
- key = int(key)
-
if not casesense and isinstance(key, str):
key = key.casefold()
From b1a1ec1540605d2ea7abdb63336ffb1c56bf6316 Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Mon, 11 Dec 2023 23:56:01 +0000
Subject: [PATCH 008/248] [ie/bitchute] Fix and improve metadata extraction
(#8507)
Closes #8492
Authored by: SirElderling
---
yt_dlp/extractor/bitchute.py | 44 ++++++++++++++++++++++++++++++++++--
1 file changed, 42 insertions(+), 2 deletions(-)
diff --git a/yt_dlp/extractor/bitchute.py b/yt_dlp/extractor/bitchute.py
index 0805b8b46..41367c5b9 100644
--- a/yt_dlp/extractor/bitchute.py
+++ b/yt_dlp/extractor/bitchute.py
@@ -7,8 +7,10 @@ from ..utils import (
ExtractorError,
OnDemandPagedList,
clean_html,
+ extract_attributes,
get_element_by_class,
get_element_by_id,
+ get_element_html_by_class,
get_elements_html_by_class,
int_or_none,
orderedSet,
@@ -17,6 +19,7 @@ from ..utils import (
traverse_obj,
unified_strdate,
urlencode_postdata,
+ urljoin,
)
@@ -34,6 +37,25 @@ class BitChuteIE(InfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'BitChute',
'upload_date': '20170103',
+ 'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
+ 'channel': 'BitChute',
+ 'channel_url': 'https://www.bitchute.com/channel/bitchute/'
+ },
+ }, {
+ # test case: video with different channel and uploader
+ 'url': 'https://www.bitchute.com/video/Yti_j9A-UZ4/',
+ 'md5': 'f10e6a8e787766235946d0868703f1d0',
+ 'info_dict': {
+ 'id': 'Yti_j9A-UZ4',
+ 'ext': 'mp4',
+ 'title': 'Israel at War | Full Measure',
+ 'description': 'md5:38cf7bc6f42da1a877835539111c69ef',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'uploader': 'sharylattkisson',
+ 'upload_date': '20231106',
+ 'uploader_url': 'https://www.bitchute.com/profile/9K0kUWA9zmd9/',
+ 'channel': 'Full Measure with Sharyl Attkisson',
+ 'channel_url': 'https://www.bitchute.com/channel/sharylattkisson/'
},
}, {
# video not downloadable in browser, but we can recover it
@@ -48,6 +70,9 @@ class BitChuteIE(InfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'BitChute',
'upload_date': '20181113',
+ 'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
+ 'channel': 'BitChute',
+ 'channel_url': 'https://www.bitchute.com/channel/bitchute/'
},
'params': {'check_formats': None},
}, {
@@ -99,6 +124,11 @@ class BitChuteIE(InfoExtractor):
reason = clean_html(get_element_by_id('page-detail', webpage)) or page_title
self.raise_geo_restricted(reason)
+ @staticmethod
+ def _make_url(html):
+ path = extract_attributes(get_element_html_by_class('spa', html) or '').get('href')
+ return urljoin('https://www.bitchute.com', path)
+
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(
@@ -121,12 +151,19 @@ class BitChuteIE(InfoExtractor):
'Video is unavailable. Please make sure this video is playable in the browser '
'before reporting this issue.', expected=True, video_id=video_id)
+ details = get_element_by_class('details', webpage) or ''
+ uploader_html = get_element_html_by_class('creator', details) or ''
+ channel_html = get_element_html_by_class('name', details) or ''
+
return {
'id': video_id,
'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
'description': self._og_search_description(webpage, default=None),
'thumbnail': self._og_search_thumbnail(webpage),
- 'uploader': clean_html(get_element_by_class('owner', webpage)),
+ 'uploader': clean_html(uploader_html),
+ 'uploader_url': self._make_url(uploader_html),
+ 'channel': clean_html(channel_html),
+ 'channel_url': self._make_url(channel_html),
'upload_date': unified_strdate(self._search_regex(
r'at \d+:\d+ UTC on (.+?)\.', publish_date, 'upload date', fatal=False)),
'formats': formats,
@@ -154,6 +191,9 @@ class BitChuteChannelIE(InfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'BitChute',
'upload_date': '20170103',
+ 'uploader_url': 'https://www.bitchute.com/profile/I5NgtHZn9vPj/',
+ 'channel': 'BitChute',
+ 'channel_url': 'https://www.bitchute.com/channel/bitchute/',
'duration': 16,
'view_count': int,
},
@@ -169,7 +209,7 @@ class BitChuteChannelIE(InfoExtractor):
'info_dict': {
'id': 'wV9Imujxasw9',
'title': 'Bruce MacDonald and "The Light of Darkness"',
- 'description': 'md5:04913227d2714af1d36d804aa2ab6b1e',
+ 'description': 'md5:747724ef404eebdfc04277714f81863e',
}
}]
From e370f9ec36972d06100a3db893b397bfc1b07b4d Mon Sep 17 00:00:00 2001
From: Benjamin Krausse
Date: Mon, 11 Dec 2023 17:52:59 -0600
Subject: [PATCH 009/248] [ie] Add `media_type` field
Authored by: trainman261
---
README.md | 1 +
yt_dlp/extractor/common.py | 1 +
2 files changed, 2 insertions(+)
diff --git a/README.md b/README.md
index 78d4799a1..06aceec02 100644
--- a/README.md
+++ b/README.md
@@ -1333,6 +1333,7 @@ The available fields are:
- `was_live` (boolean): Whether this video was originally a live stream
- `playable_in_embed` (string): Whether this video is allowed to play in embedded players on other sites
- `availability` (string): Whether the video is "private", "premium_only", "subscriber_only", "needs_auth", "unlisted" or "public"
+ - `media_type` (string): The type of media as classified by the site, e.g. "episode", "clip", "trailer"
- `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
- `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
- `extractor` (string): Name of the extractor
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index b179f4038..af534775f 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -382,6 +382,7 @@ class InfoExtractor:
'private', 'premium_only', 'subscriber_only', 'needs_auth',
'unlisted' or 'public'. Use 'InfoExtractor._availability'
to set it
+ media_type: The type of media as classified by the site, e.g. "episode", "clip", "trailer"
_old_archive_ids: A list of old archive ids needed for backward compatibility
_format_sort_fields: A list of fields to use for sorting formats
__post_extractor: A function to be called just before the metadata is
From 7e09c147fdccb44806bbf601573adc4b77210a89 Mon Sep 17 00:00:00 2001
From: trainman261
Date: Tue, 12 Dec 2023 01:00:35 +0100
Subject: [PATCH 010/248] [ie/theplatform] Extract more metadata (#8635)
Authored by: trainman261
---
yt_dlp/extractor/aenetworks.py | 23 ++++++++++++++++--
yt_dlp/extractor/cbc.py | 37 ++++++++++++++++++++++-------
yt_dlp/extractor/cwtv.py | 4 ++++
yt_dlp/extractor/mediaset.py | 2 ++
yt_dlp/extractor/nbc.py | 4 ++++
yt_dlp/extractor/scrippsnetworks.py | 2 ++
yt_dlp/extractor/theplatform.py | 12 ++++++++++
7 files changed, 73 insertions(+), 11 deletions(-)
diff --git a/yt_dlp/extractor/aenetworks.py b/yt_dlp/extractor/aenetworks.py
index cc26653c1..63a0532ef 100644
--- a/yt_dlp/extractor/aenetworks.py
+++ b/yt_dlp/extractor/aenetworks.py
@@ -121,11 +121,21 @@ class AENetworksIE(AENetworksBaseIE):
'info_dict': {
'id': '22253814',
'ext': 'mp4',
- 'title': 'Winter is Coming',
- 'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
+ 'title': 'Winter Is Coming',
+ 'description': 'md5:a40e370925074260b1c8a633c632c63a',
'timestamp': 1338306241,
'upload_date': '20120529',
'uploader': 'AENE-NEW',
+ 'duration': 2592.0,
+ 'thumbnail': r're:^https?://.*\.jpe?g$',
+ 'chapters': 'count:5',
+ 'tags': 'count:14',
+ 'categories': ['Mountain Men'],
+ 'episode_number': 1,
+ 'episode': 'Episode 1',
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'series': 'Mountain Men',
},
'params': {
# m3u8 download
@@ -143,6 +153,15 @@ class AENetworksIE(AENetworksBaseIE):
'timestamp': 1452634428,
'upload_date': '20160112',
'uploader': 'AENE-NEW',
+ 'duration': 1277.695,
+ 'thumbnail': r're:^https?://.*\.jpe?g$',
+ 'chapters': 'count:4',
+ 'tags': 'count:23',
+ 'episode': 'Episode 1',
+ 'episode_number': 1,
+ 'season': 'Season 9',
+ 'season_number': 9,
+ 'series': 'Duck Dynasty',
},
'params': {
# m3u8 download
diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py
index 29f0e307d..b5beb1ec8 100644
--- a/yt_dlp/extractor/cbc.py
+++ b/yt_dlp/extractor/cbc.py
@@ -180,6 +180,13 @@ class CBCPlayerIE(InfoExtractor):
'thumbnail': 'http://thumbnails.cbc.ca/maven_legacy/thumbnails/sonali-karnick-220.jpg',
'chapters': [],
'duration': 494.811,
+ 'categories': ['AudioMobile/All in a Weekend Montreal'],
+ 'tags': 'count:8',
+ 'location': 'Quebec',
+ 'series': 'All in a Weekend Montreal',
+ 'season': 'Season 2015',
+ 'season_number': 2015,
+ 'media_type': 'Excerpt',
},
}, {
'url': 'http://www.cbc.ca/player/play/2164402062',
@@ -195,25 +202,37 @@ class CBCPlayerIE(InfoExtractor):
'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/277/67/cancer_852x480_2164412612.jpg',
'chapters': [],
'duration': 186.867,
+ 'series': 'CBC News: Windsor at 6:00',
+ 'categories': ['News/Canada/Windsor'],
+ 'location': 'Windsor',
+ 'tags': ['cancer'],
+ 'creator': 'Allison Johnson',
+ 'media_type': 'Excerpt',
},
}, {
# Has subtitles
# These broadcasts expire after ~1 month, can find new test URL here:
# https://www.cbc.ca/player/news/TV%20Shows/The%20National/Latest%20Broadcast
- 'url': 'http://www.cbc.ca/player/play/2249992771553',
- 'md5': '2f2fb675dd4f0f8a5bb7588d1b13bacd',
+ 'url': 'http://www.cbc.ca/player/play/2284799043667',
+ 'md5': '9b49f0839e88b6ec0b01d840cf3d42b5',
'info_dict': {
- 'id': '2249992771553',
+ 'id': '2284799043667',
'ext': 'mp4',
- 'title': 'The National | Women’s soccer pay, Florida seawater, Swift quake',
- 'description': 'md5:adba28011a56cfa47a080ff198dad27a',
- 'timestamp': 1690596000,
- 'duration': 2716.333,
+ 'title': 'The National | Hockey coach charged, Green grants, Safer drugs',
+ 'description': 'md5:84ef46321c94bcf7d0159bb565d26bfa',
+ 'timestamp': 1700272800,
+ 'duration': 2718.833,
'subtitles': {'eng': [{'ext': 'vtt', 'protocol': 'm3u8_native'}]},
- 'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/481/326/thumbnail.jpeg',
+ 'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/907/171/thumbnail.jpeg',
'uploader': 'CBCC-NEW',
'chapters': 'count:5',
- 'upload_date': '20230729',
+ 'upload_date': '20231118',
+ 'categories': 'count:4',
+ 'series': 'The National - Full Show',
+ 'tags': 'count:1',
+ 'creator': 'News',
+ 'location': 'Canada',
+ 'media_type': 'Full Program',
},
}]
diff --git a/yt_dlp/extractor/cwtv.py b/yt_dlp/extractor/cwtv.py
index 9b83264ee..69d50daf6 100644
--- a/yt_dlp/extractor/cwtv.py
+++ b/yt_dlp/extractor/cwtv.py
@@ -46,6 +46,10 @@ class CWTVIE(InfoExtractor):
'timestamp': 1444107300,
'age_limit': 14,
'uploader': 'CWTV',
+ 'thumbnail': r're:^https?://.*\.jpe?g$',
+ 'chapters': 'count:4',
+ 'episode': 'Episode 20',
+ 'season': 'Season 11',
},
'params': {
# m3u8 download
diff --git a/yt_dlp/extractor/mediaset.py b/yt_dlp/extractor/mediaset.py
index 2d6204298..e04a1ce90 100644
--- a/yt_dlp/extractor/mediaset.py
+++ b/yt_dlp/extractor/mediaset.py
@@ -73,6 +73,7 @@ class MediasetIE(ThePlatformBaseIE):
'season_number': 5,
'episode_number': 5,
'chapters': [{'start_time': 0.0, 'end_time': 3409.08}, {'start_time': 3409.08, 'end_time': 6565.008}],
+ 'categories': ['Informazione'],
},
}, {
# DRM
@@ -149,6 +150,7 @@ class MediasetIE(ThePlatformBaseIE):
'season_number': 12,
'episode': 'Episode 8',
'episode_number': 8,
+ 'categories': ['Intrattenimento'],
},
'params': {
'skip_download': True,
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index 2d3aa26ec..267fa8353 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -53,6 +53,8 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'chapters': 'count:1',
'tags': 'count:4',
'thumbnail': r're:https?://.+\.jpg',
+ 'categories': ['Series/The Tonight Show Starring Jimmy Fallon'],
+ 'media_type': 'Full Episode',
},
'params': {
'skip_download': 'm3u8',
@@ -131,6 +133,8 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'tags': 'count:10',
'age_limit': 0,
'thumbnail': r're:https?://.+\.jpg',
+ 'categories': ['Series/Quantum Leap 2022'],
+ 'media_type': 'Highlight',
},
'params': {
'skip_download': 'm3u8',
diff --git a/yt_dlp/extractor/scrippsnetworks.py b/yt_dlp/extractor/scrippsnetworks.py
index 7f0bc9645..3912f7786 100644
--- a/yt_dlp/extractor/scrippsnetworks.py
+++ b/yt_dlp/extractor/scrippsnetworks.py
@@ -114,6 +114,8 @@ class ScrippsNetworksIE(InfoExtractor):
'timestamp': 1475678834,
'upload_date': '20161005',
'uploader': 'SCNI-SCND',
+ 'tags': 'count:10',
+ 'creator': 'Cooking Channel',
'duration': 29.995,
'chapters': [{'start_time': 0.0, 'end_time': 29.995, 'title': ''}],
'thumbnail': 'https://images.dds.discovery.com/up/tp/Scripps_-_Food_Category_Prod/122/987/0260338_630x355.jpg',
diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py
index 433ce8427..9160f5ec6 100644
--- a/yt_dlp/extractor/theplatform.py
+++ b/yt_dlp/extractor/theplatform.py
@@ -104,6 +104,10 @@ class ThePlatformBaseIE(OnceIE):
_add_chapter(chapter.get('startTime'), chapter.get('endTime'))
_add_chapter(tp_chapters[-1].get('startTime'), tp_chapters[-1].get('endTime') or duration)
+ def extract_site_specific_field(field):
+ # A number of sites have custom-prefixed keys, e.g. 'cbc$seasonNumber'
+ return traverse_obj(info, lambda k, v: v and k.endswith(f'${field}'), get_all=False)
+
return {
'title': info['title'],
'subtitles': subtitles,
@@ -113,6 +117,14 @@ class ThePlatformBaseIE(OnceIE):
'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
'uploader': info.get('billingCode'),
'chapters': chapters,
+ 'creator': traverse_obj(info, ('author', {str})) or None,
+ 'categories': traverse_obj(info, (
+ 'categories', lambda _, v: v.get('label') in ('category', None), 'name', {str})) or None,
+ 'tags': traverse_obj(info, ('keywords', {lambda x: re.split(r'[;,]\s?', x) if x else None})),
+ 'location': extract_site_specific_field('region'),
+ 'series': extract_site_specific_field('show'),
+ 'season_number': int_or_none(extract_site_specific_field('seasonNumber')),
+ 'media_type': extract_site_specific_field('programmingType') or extract_site_specific_field('type'),
}
def _extract_theplatform_metadata(self, path, video_id):
From d5d1517e7d838500800d193ac3234b06e89654cd Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Tue, 12 Dec 2023 08:29:36 +0800
Subject: [PATCH 011/248] [ie/eplus] Add login support and DRM detection
(#8661)
Authored by: pzhlkj6612
---
yt_dlp/extractor/eplus.py | 98 +++++++++++++++++++++++++++++++++++++--
1 file changed, 93 insertions(+), 5 deletions(-)
diff --git a/yt_dlp/extractor/eplus.py b/yt_dlp/extractor/eplus.py
index 3ebdcf5fb..6383691a1 100644
--- a/yt_dlp/extractor/eplus.py
+++ b/yt_dlp/extractor/eplus.py
@@ -1,15 +1,20 @@
+import json
+
from .common import InfoExtractor
from ..utils import (
ExtractorError,
try_call,
unified_timestamp,
+ urlencode_postdata,
)
class EplusIbIE(InfoExtractor):
- IE_NAME = 'eplus:inbound'
- IE_DESC = 'e+ (イープラス) overseas'
- _VALID_URL = r'https?://live\.eplus\.jp/ex/player\?ib=(?P(?:\w|%2B|%2F){86}%3D%3D)'
+ _NETRC_MACHINE = 'eplus'
+ IE_NAME = 'eplus'
+ IE_DESC = 'e+ (イープラス)'
+ _VALID_URL = [r'https?://live\.eplus\.jp/ex/player\?ib=(?P(?:\w|%2B|%2F){86}%3D%3D)',
+ r'https?://live\.eplus\.jp/(?Psample|\d+)']
_TESTS = [{
'url': 'https://live.eplus.jp/ex/player?ib=YEFxb3Vyc2Dombnjg7blkrLlrablnJLjgrnjgq%2Fjg7zjg6vjgqLjgqTjg4njg6vlkIzlpb3kvJpgTGllbGxhIQ%3D%3D',
'info_dict': {
@@ -29,14 +34,97 @@ class EplusIbIE(InfoExtractor):
'No video formats found!',
'Requested format is not available',
],
+ }, {
+ 'url': 'https://live.eplus.jp/sample',
+ 'info_dict': {
+ 'id': 'stream1ng20210719-test-005',
+ 'title': 'Online streaming test for DRM',
+ 'live_status': 'was_live',
+ 'release_date': '20210719',
+ 'release_timestamp': 1626703200,
+ 'description': None,
+ },
+ 'params': {
+ 'skip_download': True,
+ 'ignore_no_formats_error': True,
+ },
+ 'expected_warnings': [
+ 'Could not find the playlist URL. This event may not be accessible',
+ 'No video formats found!',
+ 'Requested format is not available',
+ 'This video is DRM protected',
+ ],
+ }, {
+ 'url': 'https://live.eplus.jp/2053935',
+ 'info_dict': {
+ 'id': '331320-0001-001',
+ 'title': '丘みどり2020配信LIVE Vol.2 ~秋麗~ 【Streaming+(配信チケット)】',
+ 'live_status': 'was_live',
+ 'release_date': '20200920',
+ 'release_timestamp': 1600596000,
+ },
+ 'params': {
+ 'skip_download': True,
+ 'ignore_no_formats_error': True,
+ },
+ 'expected_warnings': [
+ 'Could not find the playlist URL. This event may not be accessible',
+ 'No video formats found!',
+ 'Requested format is not available',
+ ],
}]
+ _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
+
+ def _login(self, username, password, urlh):
+ if not self._get_cookies('https://live.eplus.jp/').get('ci_session'):
+ raise ExtractorError('Unable to get ci_session cookie')
+
+ cltft_token = urlh.headers.get('X-CLTFT-Token')
+ if not cltft_token:
+ raise ExtractorError('Unable to get X-CLTFT-Token')
+ self._set_cookie('live.eplus.jp', 'X-CLTFT-Token', cltft_token)
+
+ login_json = self._download_json(
+ 'https://live.eplus.jp/member/api/v1/FTAuth/idpw', None,
+ note='Sending pre-login info', errnote='Unable to send pre-login info', headers={
+ 'Content-Type': 'application/json; charset=UTF-8',
+ 'Referer': urlh.url,
+ 'X-Cltft-Token': cltft_token,
+ 'Accept': '*/*',
+ }, data=json.dumps({
+ 'loginId': username,
+ 'loginPassword': password,
+ }).encode())
+ if not login_json.get('isSuccess'):
+ raise ExtractorError('Login failed: Invalid id or password', expected=True)
+
+ self._request_webpage(
+ urlh.url, None, note='Logging in', errnote='Unable to log in',
+ data=urlencode_postdata({
+ 'loginId': username,
+ 'loginPassword': password,
+ 'Token.Default': cltft_token,
+ 'op': 'nextPage',
+ }), headers={'Referer': urlh.url})
+
def _real_extract(self, url):
video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
+ webpage, urlh = self._download_webpage_handle(
+ url, video_id, headers={'User-Agent': self._USER_AGENT})
+ if urlh.url.startswith('https://live.eplus.jp/member/auth'):
+ username, password = self._get_login_info()
+ if not username:
+ self.raise_login_required()
+ self._login(username, password, urlh)
+ webpage = self._download_webpage(
+ url, video_id, headers={'User-Agent': self._USER_AGENT})
data_json = self._search_json(r'', webpage)]
post = traverse_obj(post_data, (
..., 'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
-
- automatic_captions, subtitles = {}, {}
- subs_data = traverse_obj(post, (..., 'video', ..., 'attachments', ..., lambda k, v: (
- k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')))
- is_video_broadcast = get_first(subs_data, 'is_video_broadcast', expected_type=bool)
- captions = get_first(subs_data, 'video_available_captions_locales', 'captions_url')
- if url_or_none(captions): # if subs_data only had a 'captions_url'
- locale = self._html_search_meta(['og:locale', 'twitter:locale'], webpage, 'locale', default='en_US')
- subtitles[locale] = [{'url': captions}]
- # or else subs_data had 'video_available_captions_locales', a list of dicts
- for caption in traverse_obj(captions, (
- {lambda x: sorted(x, key=lambda c: c['locale'])}, lambda _, v: v['captions_url'])
- ):
- lang = caption.get('localized_language') or ''
- subs = {
- 'url': caption['captions_url'],
- 'name': format_field(caption, 'localized_country', f'{lang} (%s)', default=lang),
- }
- if caption.get('localized_creation_method') or is_video_broadcast:
- automatic_captions.setdefault(caption['locale'], []).append(subs)
- else:
- subtitles.setdefault(caption['locale'], []).append(subs)
-
media = traverse_obj(post, (..., 'attachments', ..., lambda k, v: (
k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
title = get_first(media, ('title', 'text'))
description = get_first(media, ('creation_story', 'comet_sections', 'message', 'story', 'message', 'text'))
- uploader_data = (
- get_first(media, ('owner', {dict}))
- or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
- or get_first(post, ('node', 'actors', ..., {dict})) or {})
-
page_title = title or self._html_search_regex((
r']*class="uiHeaderTitle"[^>]*>(?P[^<]*)
',
r'(?s)(?P.*?)',
@@ -513,11 +498,15 @@ class FacebookIE(InfoExtractor):
description = description or self._html_search_meta(
['description', 'og:description', 'twitter:description'],
webpage, 'description', default=None)
+ uploader_data = (
+ get_first(media, ('owner', {dict}))
+ or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
+ or get_first(post, ('node', 'actors', ..., {dict}))
+ or get_first(post, ('event', 'event_creator', {dict})) or {})
uploader = uploader_data.get('name') or (
clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
or self._search_regex(
(r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes('title')), webpage, 'uploader', fatal=False))
-
timestamp = int_or_none(self._search_regex(
r']+data-utime=["\'](\d+)', webpage,
'timestamp', default=None))
@@ -539,8 +528,6 @@ class FacebookIE(InfoExtractor):
webpage, 'view count', default=None)),
'concurrent_view_count': get_first(post, (
('video', (..., ..., 'attachments', ..., 'media')), 'liveViewerCount', {int_or_none})),
- 'automatic_captions': automatic_captions,
- 'subtitles': subtitles,
}
info_json_ld = self._search_json_ld(webpage, video_id, default={})
@@ -638,6 +625,29 @@ class FacebookIE(InfoExtractor):
'url': playable_url,
})
extract_dash_manifest(video, formats)
+
+ automatic_captions, subtitles = {}, {}
+ is_broadcast = traverse_obj(video, ('is_video_broadcast', {bool}))
+ for caption in traverse_obj(video, (
+ 'video_available_captions_locales',
+ {lambda x: sorted(x, key=lambda c: c['locale'])},
+ lambda _, v: url_or_none(v['captions_url'])
+ )):
+ lang = caption.get('localized_language') or 'und'
+ subs = {
+ 'url': caption['captions_url'],
+ 'name': format_field(caption, 'localized_country', f'{lang} (%s)', default=lang),
+ }
+ if caption.get('localized_creation_method') or is_broadcast:
+ automatic_captions.setdefault(caption['locale'], []).append(subs)
+ else:
+ subtitles.setdefault(caption['locale'], []).append(subs)
+ captions_url = traverse_obj(video, ('captions_url', {url_or_none}))
+ if captions_url and not automatic_captions and not subtitles:
+ locale = self._html_search_meta(
+ ['og:locale', 'twitter:locale'], webpage, 'locale', default='en_US')
+ (automatic_captions if is_broadcast else subtitles)[locale] = [{'url': captions_url}]
+
info = {
'id': v_id,
'formats': formats,
@@ -647,6 +657,8 @@ class FacebookIE(InfoExtractor):
'timestamp': traverse_obj(video, 'publish_time', 'creation_time', expected_type=int_or_none),
'duration': (float_or_none(video.get('playable_duration_in_ms'), 1000)
or float_or_none(video.get('length_in_second'))),
+ 'automatic_captions': automatic_captions,
+ 'subtitles': subtitles,
}
process_formats(info)
description = try_get(video, lambda x: x['savable_description']['text'])
@@ -681,7 +693,8 @@ class FacebookIE(InfoExtractor):
for edge in edges:
parse_attachment(edge, key='node')
- video = data.get('video') or {}
+ video = traverse_obj(data, (
+ 'event', 'cover_media_renderer', 'cover_video'), 'video', expected_type=dict) or {}
if video:
attachments = try_get(video, [
lambda x: x['story']['attachments'],
From 67bb70cd700c8d4c3149cd9e0539a5f32c3d1ce6 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Mon, 29 Jan 2024 21:16:46 +0100
Subject: [PATCH 105/248] [ie/Vbox7] Fix extractor (#9100)
Closes #1098, Closes #5661
Authored by: seproDev
---
yt_dlp/extractor/vbox7.py | 82 ++++++++++++++++++++-------------------
1 file changed, 43 insertions(+), 39 deletions(-)
diff --git a/yt_dlp/extractor/vbox7.py b/yt_dlp/extractor/vbox7.py
index be35dad1c..21bf4232b 100644
--- a/yt_dlp/extractor/vbox7.py
+++ b/yt_dlp/extractor/vbox7.py
@@ -1,5 +1,6 @@
from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import ExtractorError, base_url, int_or_none, url_basename
+from ..utils.traversal import traverse_obj
class Vbox7IE(InfoExtractor):
@@ -19,7 +20,7 @@ class Vbox7IE(InfoExtractor):
_GEO_COUNTRIES = ['BG']
_TESTS = [{
'url': 'http://vbox7.com/play:0946fff23c',
- 'md5': 'a60f9ab3a3a2f013ef9a967d5f7be5bf',
+ 'md5': '50ca1f78345a9c15391af47d8062d074',
'info_dict': {
'id': '0946fff23c',
'ext': 'mp4',
@@ -29,19 +30,25 @@ class Vbox7IE(InfoExtractor):
'timestamp': 1470982814,
'upload_date': '20160812',
'uploader': 'zdraveibulgaria',
- },
- 'params': {
- 'proxy': '127.0.0.1:8118',
+ 'view_count': int,
+ 'duration': 2640,
},
}, {
'url': 'http://vbox7.com/play:249bb972c2',
- 'md5': '99f65c0c9ef9b682b97313e052734c3f',
+ 'md5': 'da1dd2eb245200cb86e6d09d43232116',
'info_dict': {
'id': '249bb972c2',
'ext': 'mp4',
'title': 'Смях! Чудо - чист за секунди - Скрита камера',
+ 'uploader': 'svideteliat_ot_varshava',
+ 'view_count': int,
+ 'timestamp': 1360215023,
+ 'thumbnail': 'https://i49.vbox7.com/design/iconci/png/noimg6.png',
+ 'description': 'Смях! Чудо - чист за секунди - Скрита камера',
+ 'upload_date': '20130207',
+ 'duration': 83,
},
- 'skip': 'georestricted',
+ 'expected_warnings': ['Failed to download m3u8 information'],
}, {
'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
'only_matching': True,
@@ -53,41 +60,38 @@ class Vbox7IE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- response = self._download_json(
- 'https://www.vbox7.com/ajax/video/nextvideo.php?vid=%s' % video_id,
- video_id)
-
- if 'error' in response:
- raise ExtractorError(
- '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
-
- video = response['options']
-
- title = video['title']
- video_url = video['src']
-
- if '/na.mp4' in video_url:
- self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+ data = self._download_json(
+ 'https://www.vbox7.com/aj/player/item/options', video_id,
+ query={'vid': video_id})['options']
- uploader = video.get('uploader')
+ src_url = data.get('src')
+ if src_url in (None, '', 'blank'):
+ raise ExtractorError('Video is unavailable', expected=True)
- webpage = self._download_webpage(
- 'http://vbox7.com/play:%s' % video_id, video_id, fatal=None)
+ fmt_base = url_basename(src_url).rsplit('.', 1)[0].rsplit('_', 1)[0]
+ if fmt_base == 'vn':
+ self.raise_geo_restricted()
- info = {}
+ fmt_base = base_url(src_url) + fmt_base
- if webpage:
- info = self._search_json_ld(
- webpage.replace('"/*@context"', '"@context"'), video_id,
- fatal=False)
+ formats = self._extract_m3u8_formats(
+ f'{fmt_base}.m3u8', video_id, m3u8_id='hls', fatal=False)
+ # TODO: Add MPD formats, when dash range support is added
+ for res in traverse_obj(data, ('resolutions', lambda _, v: v != 0, {int})):
+ formats.append({
+ 'url': f'{fmt_base}_{res}.mp4',
+ 'format_id': f'http-{res}',
+ 'height': res,
+ })
- info.update({
+ return {
'id': video_id,
- 'title': title,
- 'url': video_url,
- 'uploader': uploader,
- 'thumbnail': self._proto_relative_url(
- info.get('thumbnail') or self._og_search_thumbnail(webpage),
- 'http:'),
- })
- return info
+ 'formats': formats,
+ **self._search_json_ld(self._download_webpage(
+ f'https://www.vbox7.com/play:{video_id}', video_id, fatal=False) or '', video_id, fatal=False),
+ **traverse_obj(data, {
+ 'title': ('title', {str}),
+ 'uploader': ('uploader', {str}),
+ 'duration': ('duration', {int_or_none}),
+ }),
+ }
From 3725b4f0c93ca3943e6300013a9670e4ab757fda Mon Sep 17 00:00:00 2001
From: Simon Sawicki
Date: Wed, 31 Jan 2024 09:35:35 +0100
Subject: [PATCH 106/248] [core] Add `--compat-options 2023` (#9084)
Authored by: Grub4K
---
README.md | 3 ++-
yt_dlp/options.py | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index b6a79667c..7dc3bb2f6 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,8 @@ For ease of use, a few more compat options are available:
* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx`
* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx`
* `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date`
-* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options
+* `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress`
+* `--compat-options 2023`: Same as `--compat-options prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options
# INSTALLATION
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index e9d927717..9bea6549d 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -476,7 +476,8 @@ def create_parser():
'youtube-dl': ['all', '-multistreams', '-playlist-match-filter', '-manifest-filesize-approx'],
'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter', '-manifest-filesize-approx'],
'2021': ['2022', 'no-certifi', 'filename-sanitization', 'no-youtube-prefer-utc-upload-date'],
- '2022': ['no-external-downloader-progress', 'playlist-match-filter', 'prefer-legacy-http-handler', 'manifest-filesize-approx'],
+ '2022': ['2023', 'no-external-downloader-progress', 'playlist-match-filter'],
+ '2023': ['prefer-legacy-http-handler', 'manifest-filesize-approx'],
}
}, help=(
'Options that can help keep compatibility with youtube-dl or youtube-dlc '
From cbed249aaa053a3f425b9bafc97f8dbd71c44487 Mon Sep 17 00:00:00 2001
From: Simon Sawicki
Date: Wed, 31 Jan 2024 09:43:52 +0100
Subject: [PATCH 107/248] [cookies] Fix `--cookies-from-browser` for `snap`
Firefox (#9016)
Authored by: Grub4K
---
yt_dlp/cookies.py | 46 ++++++++++++++++++++++++++++++----------------
1 file changed, 30 insertions(+), 16 deletions(-)
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index eac033e39..a92ab4164 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -1,6 +1,7 @@
import base64
import collections
import contextlib
+import glob
import http.cookiejar
import http.cookies
import io
@@ -122,13 +123,14 @@ def _extract_firefox_cookies(profile, container, logger):
return YoutubeDLCookieJar()
if profile is None:
- search_root = _firefox_browser_dir()
+ search_roots = list(_firefox_browser_dirs())
elif _is_path(profile):
- search_root = profile
+ search_roots = [profile]
else:
- search_root = os.path.join(_firefox_browser_dir(), profile)
+ search_roots = [os.path.join(path, profile) for path in _firefox_browser_dirs()]
+ search_root = ', '.join(map(repr, search_roots))
- cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger)
+ cookie_database_path = _newest(_firefox_cookie_dbs(search_roots))
if cookie_database_path is None:
raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
@@ -182,12 +184,21 @@ def _extract_firefox_cookies(profile, container, logger):
cursor.connection.close()
-def _firefox_browser_dir():
+def _firefox_browser_dirs():
if sys.platform in ('cygwin', 'win32'):
- return os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
+ yield os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
+
elif sys.platform == 'darwin':
- return os.path.expanduser('~/Library/Application Support/Firefox/Profiles')
- return os.path.expanduser('~/.mozilla/firefox')
+ yield os.path.expanduser('~/Library/Application Support/Firefox/Profiles')
+
+ else:
+ yield from map(os.path.expanduser, ('~/.mozilla/firefox', '~/snap/firefox/common/.mozilla/firefox'))
+
+
+def _firefox_cookie_dbs(roots):
+ for root in map(os.path.abspath, roots):
+ for pattern in ('', '*/', 'Profiles/*/'):
+ yield from glob.iglob(os.path.join(root, pattern, 'cookies.sqlite'))
def _get_chromium_based_browser_settings(browser_name):
@@ -268,7 +279,7 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
logger.error(f'{browser_name} does not support profiles')
search_root = config['browser_dir']
- cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies', logger)
+ cookie_database_path = _newest(_find_files(search_root, 'Cookies', logger))
if cookie_database_path is None:
raise FileNotFoundError(f'could not find {browser_name} cookies database in "{search_root}"')
logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
@@ -947,7 +958,7 @@ def _get_windows_v10_key(browser_root, logger):
References:
- [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
"""
- path = _find_most_recently_used_file(browser_root, 'Local State', logger)
+ path = _newest(_find_files(browser_root, 'Local State', logger))
if path is None:
logger.error('could not find local state file')
return None
@@ -1049,17 +1060,20 @@ def _get_column_names(cursor, table_name):
return [row[1].decode() for row in table_info]
-def _find_most_recently_used_file(root, filename, logger):
+def _newest(files):
+ return max(files, key=lambda path: os.lstat(path).st_mtime, default=None)
+
+
+def _find_files(root, filename, logger):
# if there are multiple browser profiles, take the most recently used one
- i, paths = 0, []
+ i = 0
with _create_progress_bar(logger) as progress_bar:
- for curr_root, dirs, files in os.walk(root):
+ for curr_root, _, files in os.walk(root):
for file in files:
i += 1
progress_bar.print(f'Searching for "{filename}": {i: 6d} files searched')
if file == filename:
- paths.append(os.path.join(curr_root, file))
- return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
+ yield os.path.join(curr_root, file)
def _merge_cookie_jars(jars):
@@ -1073,7 +1087,7 @@ def _merge_cookie_jars(jars):
def _is_path(value):
- return os.path.sep in value
+ return any(sep in value for sep in (os.path.sep, os.path.altsep) if sep)
def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None):
From 2792092afd367e39251ace1fb2819c855ab8919f Mon Sep 17 00:00:00 2001
From: Simon Sawicki
Date: Wed, 31 Jan 2024 09:56:14 +0100
Subject: [PATCH 108/248] [cookies] Improve error message for Windows
`--cookies-from-browser chrome` issue (#9080)
Authored by: Grub4K
---
yt_dlp/cookies.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index a92ab4164..deb2e35f2 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -24,7 +24,8 @@ from .aes import (
aes_gcm_decrypt_and_verify_bytes,
unpad_pkcs7,
)
-from .compat import functools
+from .compat import functools # isort: split
+from .compat import compat_os_name
from .dependencies import (
_SECRETSTORAGE_UNAVAILABLE_REASON,
secretstorage,
@@ -32,6 +33,7 @@ from .dependencies import (
)
from .minicurses import MultilinePrinter, QuietMultilinePrinter
from .utils import (
+ DownloadError,
Popen,
error_to_str,
expand_path,
@@ -318,6 +320,12 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
counts['unencrypted'] = unencrypted_cookies
logger.debug(f'cookie version breakdown: {counts}')
return jar
+ except PermissionError as error:
+ if compat_os_name == 'nt' and error.errno == 13:
+ message = 'Could not copy Chrome cookie database. See https://github.com/yt-dlp/yt-dlp/issues/7271 for more info'
+ logger.error(message)
+ raise DownloadError(message) # force exit
+ raise
finally:
if cursor is not None:
cursor.connection.close()
From d63eae7e7ffb1f3e733e552b9e5e82355bfba214 Mon Sep 17 00:00:00 2001
From: bashonly
Date: Wed, 31 Jan 2024 03:11:41 -0600
Subject: [PATCH 109/248] [core] Don't select storyboard formats as fallback
Closes #7715
Authored by: bashonly
---
yt_dlp/YoutubeDL.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 5dcefb5b8..e7d654d0f 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2451,7 +2451,7 @@ class YoutubeDL:
# for extractors with incomplete formats (audio only (soundcloud)
# or video only (imgur)) best/worst will fallback to
# best/worst {video,audio}-only format
- matches = formats
+ matches = list(filter(lambda f: f.get('vcodec') != 'none' or f.get('acodec') != 'none', formats))
elif seperate_fallback and not ctx['has_merged_format']:
# for compatibility with youtube-dl when there is no pre-merged format
matches = list(filter(seperate_fallback, formats))
From 62c65bfaf81e04e6746f6fdbafe384eb3edddfbc Mon Sep 17 00:00:00 2001
From: Radu Manole
Date: Wed, 31 Jan 2024 19:41:31 +0200
Subject: [PATCH 110/248] [ie/NinaProtocol] Add extractor (#8946)
Closes #8709, Closes #8764
Authored by: RaduManole, seproDev
Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
yt_dlp/extractor/_extractors.py | 1 +
yt_dlp/extractor/ninaprotocol.py | 225 +++++++++++++++++++++++++++++++
2 files changed, 226 insertions(+)
create mode 100644 yt_dlp/extractor/ninaprotocol.py
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index f8488d304..69deaf15a 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1284,6 +1284,7 @@ from .niconico import (
NicovideoTagURLIE,
NiconicoLiveIE,
)
+from .ninaprotocol import NinaProtocolIE
from .ninecninemedia import (
NineCNineMediaIE,
CPTwentyFourIE,
diff --git a/yt_dlp/extractor/ninaprotocol.py b/yt_dlp/extractor/ninaprotocol.py
new file mode 100644
index 000000000..ea57c5f38
--- /dev/null
+++ b/yt_dlp/extractor/ninaprotocol.py
@@ -0,0 +1,225 @@
+from .common import InfoExtractor
+from ..utils import int_or_none, mimetype2ext, parse_iso8601, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class NinaProtocolIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?ninaprotocol\.com/releases/(?P[^/#?]+)'
+ _TESTS = [{
+ 'url': 'https://www.ninaprotocol.com/releases/3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
+ 'info_dict': {
+ 'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
+ 'title': 'The Spatulas - March Chant',
+ 'tags': ['punk', 'postpresentmedium', 'cambridge'],
+ 'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+ 'channel': 'ppm',
+ 'description': 'md5:bb9f9d39d8f786449cd5d0ff7c5772db',
+ 'album': 'The Spatulas - March Chant',
+ 'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+ 'timestamp': 1701417610,
+ 'uploader': 'ppmrecs',
+ 'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+ 'display_id': 'the-spatulas-march-chant',
+ 'upload_date': '20231201',
+ 'album_artist': 'Post Present Medium ',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ 'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_1',
+ 'title': 'March Chant In April',
+ 'track': 'March Chant In April',
+ 'ext': 'mp3',
+ 'duration': 152,
+ 'track_number': 1,
+ 'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+ 'uploader': 'ppmrecs',
+ 'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+ 'timestamp': 1701417610,
+ 'channel': 'ppm',
+ 'album': 'The Spatulas - March Chant',
+ 'tags': ['punk', 'postpresentmedium', 'cambridge'],
+ 'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+ 'upload_date': '20231201',
+ 'album_artist': 'Post Present Medium ',
+ }
+ }, {
+ 'info_dict': {
+ 'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_2',
+ 'title': 'Rescue Mission',
+ 'track': 'Rescue Mission',
+ 'ext': 'mp3',
+ 'duration': 212,
+ 'track_number': 2,
+ 'album_artist': 'Post Present Medium ',
+ 'uploader': 'ppmrecs',
+ 'tags': ['punk', 'postpresentmedium', 'cambridge'],
+ 'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+ 'channel': 'ppm',
+ 'upload_date': '20231201',
+ 'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+ 'timestamp': 1701417610,
+ 'album': 'The Spatulas - March Chant',
+ 'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+ }
+ }, {
+ 'info_dict': {
+ 'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_3',
+ 'title': 'Slinger Style',
+ 'track': 'Slinger Style',
+ 'ext': 'mp3',
+ 'duration': 179,
+ 'track_number': 3,
+ 'timestamp': 1701417610,
+ 'upload_date': '20231201',
+ 'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+ 'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+ 'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+ 'album_artist': 'Post Present Medium ',
+ 'album': 'The Spatulas - March Chant',
+ 'tags': ['punk', 'postpresentmedium', 'cambridge'],
+ 'uploader': 'ppmrecs',
+ 'channel': 'ppm',
+ }
+ }, {
+ 'info_dict': {
+ 'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_4',
+ 'title': 'Psychic Signal',
+ 'track': 'Psychic Signal',
+ 'ext': 'mp3',
+ 'duration': 220,
+ 'track_number': 4,
+ 'tags': ['punk', 'postpresentmedium', 'cambridge'],
+ 'upload_date': '20231201',
+ 'album': 'The Spatulas - March Chant',
+ 'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+ 'timestamp': 1701417610,
+ 'album_artist': 'Post Present Medium ',
+ 'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+ 'channel': 'ppm',
+ 'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+ 'uploader': 'ppmrecs',
+ }
+ }, {
+ 'info_dict': {
+ 'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_5',
+ 'title': 'Curvy Color',
+ 'track': 'Curvy Color',
+ 'ext': 'mp3',
+ 'duration': 148,
+ 'track_number': 5,
+ 'timestamp': 1701417610,
+ 'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+ 'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+ 'album': 'The Spatulas - March Chant',
+ 'album_artist': 'Post Present Medium ',
+ 'channel': 'ppm',
+ 'tags': ['punk', 'postpresentmedium', 'cambridge'],
+ 'uploader': 'ppmrecs',
+ 'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+ 'upload_date': '20231201',
+ }
+ }, {
+ 'info_dict': {
+ 'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_6',
+ 'title': 'Caveman Star',
+ 'track': 'Caveman Star',
+ 'ext': 'mp3',
+ 'duration': 121,
+ 'track_number': 6,
+ 'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
+ 'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
+ 'tags': ['punk', 'postpresentmedium', 'cambridge'],
+ 'album_artist': 'Post Present Medium ',
+ 'uploader': 'ppmrecs',
+ 'timestamp': 1701417610,
+ 'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
+ 'album': 'The Spatulas - March Chant',
+ 'channel': 'ppm',
+ 'upload_date': '20231201',
+ },
+ }],
+ }, {
+ 'url': 'https://www.ninaprotocol.com/releases/f-g-s-american-shield',
+ 'info_dict': {
+ 'id': '76PZnJwaMgViQHYfA4NYJXds7CmW6vHQKAtQUxGene6J',
+ 'description': 'md5:63f08d5db558b4b36e1896f317062721',
+ 'title': 'F.G.S. - American Shield',
+ 'uploader_id': 'Ej3rozs11wYqFk1Gs6oggGCkGLz8GzBhmJfnUxf6gPci',
+ 'channel_id': '6JuksCZPXuP16wJ1BUfwuukJzh42C7guhLrFPPkVJfyE',
+ 'channel': 'tinkscough',
+ 'tags': [],
+ 'album_artist': 'F.G.S.',
+ 'album': 'F.G.S. - American Shield',
+ 'thumbnail': 'https://www.arweave.net/YJpgImkXLT9SbpFb576KuZ5pm6bdvs452LMs3Rx6lm8',
+ 'display_id': 'f-g-s-american-shield',
+ 'uploader': 'flannerysilva',
+ 'timestamp': 1702395858,
+ 'upload_date': '20231212',
+ },
+ 'playlist_count': 1,
+ }, {
+ 'url': 'https://www.ninaprotocol.com/releases/time-to-figure-things-out',
+ 'info_dict': {
+ 'id': '6Zi1nC5hj6b13NkpxVYwRhFy6mYA7oLBbe9DMrgGDcYh',
+ 'display_id': 'time-to-figure-things-out',
+ 'description': 'md5:960202ed01c3134bb8958f1008527e35',
+ 'timestamp': 1706283607,
+ 'title': 'DJ STEPDAD - time to figure things out',
+ 'album_artist': 'DJ STEPDAD',
+ 'uploader': 'tddvsss',
+ 'upload_date': '20240126',
+ 'album': 'time to figure things out',
+ 'uploader_id': 'AXQNRgTyYsySyAMFDwxzumuGjfmoXshorCesjpquwCBi',
+ 'thumbnail': 'https://www.arweave.net/O4i8bcKVqJVZvNeHHFp6r8knpFGh9ZwEgbeYacr4nss',
+ 'tags': [],
+ },
+ 'playlist_count': 4,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ release = self._download_json(
+ f'https://api.ninaprotocol.com/v1/releases/{video_id}', video_id)['release']
+
+ video_id = release.get('publicKey') or video_id
+
+ common_info = traverse_obj(release, {
+ 'album': ('metadata', 'properties', 'title', {str}),
+ 'album_artist': ((('hub', 'data'), 'publisherAccount'), 'displayName', {str}),
+ 'timestamp': ('datetime', {parse_iso8601}),
+ 'thumbnail': ('metadata', 'image', {url_or_none}),
+ 'uploader': ('publisherAccount', 'handle', {str}),
+ 'uploader_id': ('publisherAccount', 'publicKey', {str}),
+ 'channel': ('hub', 'handle', {str}),
+ 'channel_id': ('hub', 'publicKey', {str}),
+ }, get_all=False)
+ common_info['tags'] = traverse_obj(release, ('metadata', 'properties', 'tags', ..., {str}))
+
+ entries = []
+ for track_num, track in enumerate(traverse_obj(release, (
+ 'metadata', 'properties', 'files', lambda _, v: url_or_none(v['uri']))), 1):
+ entries.append({
+ 'id': f'{video_id}_{track_num}',
+ 'url': track['uri'],
+ **traverse_obj(track, {
+ 'title': ('track_title', {str}),
+ 'track': ('track_title', {str}),
+ 'ext': ('type', {mimetype2ext}),
+ 'track_number': ('track', {int_or_none}),
+ 'duration': ('duration', {int_or_none}),
+ }),
+ 'vcodec': 'none',
+ **common_info,
+ })
+
+ return {
+ '_type': 'playlist',
+ 'id': video_id,
+ 'entries': entries,
+ **traverse_obj(release, {
+ 'display_id': ('slug', {str}),
+ 'title': ('metadata', 'name', {str}),
+ 'description': ('metadata', 'description', {str}),
+ }),
+ **common_info,
+ }
From 4a6ff0b47a700dee3ee5c54804c31965308479ae Mon Sep 17 00:00:00 2001
From: jazz1611
Date: Thu, 1 Feb 2024 00:56:29 +0700
Subject: [PATCH 111/248] [ie/redtube] Support redtube.com.br URLs (#9103)
Authored by: jazz1611
---
yt_dlp/extractor/redtube.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/yt_dlp/extractor/redtube.py b/yt_dlp/extractor/redtube.py
index 36d530daf..965abbee8 100644
--- a/yt_dlp/extractor/redtube.py
+++ b/yt_dlp/extractor/redtube.py
@@ -12,7 +12,7 @@ from ..utils import (
class RedTubeIE(InfoExtractor):
- _VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P[0-9]+)'
+ _VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com(?:\.br)?/|embed\.redtube\.com/\?.*?\bid=)(?P[0-9]+)'
_EMBED_REGEX = [r'
', webpage, 'author', default=None),
+ }
-class NYTimesCookingIE(NYTimesBaseIE):
- _VALID_URL = r'https?://cooking\.nytimes\.com/(?:guid|recip)es/(?P\d+)'
+
+class NYTimesCookingRecipeIE(InfoExtractor):
+ _VALID_URL = r'https?://cooking\.nytimes\.com/recipes/(?P\d+)'
_TESTS = [{
'url': 'https://cooking.nytimes.com/recipes/1017817-cranberry-curd-tart',
- 'md5': 'dab81fa2eaeb3f9ed47498bdcfcdc1d3',
+ 'md5': '579e83bbe8e61e9de67f80edba8a78a8',
'info_dict': {
- 'id': '100000004756089',
- 'ext': 'mov',
- 'timestamp': 1479383008,
- 'uploader': 'By SHAW LASH, ADAM SAEWITZ and JAMES HERRON',
- 'title': 'Cranberry Tart',
- 'upload_date': '20161117',
- 'description': 'If you are a fan of lemon curd or the classic French tarte au citron, you will love this cranberry version.',
+ 'id': '1017817',
+ 'ext': 'mp4',
+ 'title': 'Cranberry Curd Tart',
+ 'description': 'md5:ad77a3fc321db636256d4343c5742152',
+ 'timestamp': 1447804800,
+ 'upload_date': '20151118',
+ 'creator': 'David Tanis',
+ 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
},
}, {
- 'url': 'https://cooking.nytimes.com/guides/13-how-to-cook-a-turkey',
- 'md5': '4b2e8c70530a89b8d905a2b572316eb8',
+ 'url': 'https://cooking.nytimes.com/recipes/1024781-neapolitan-checkerboard-cookies',
+ 'md5': '58df35998241dcf0620e99e646331b42',
'info_dict': {
- 'id': '100000003951728',
- 'ext': 'mov',
- 'timestamp': 1445509539,
- 'description': 'Turkey guide',
- 'upload_date': '20151022',
- 'title': 'Turkey',
- }
+ 'id': '1024781',
+ 'ext': 'mp4',
+ 'title': 'Neapolitan Checkerboard Cookies',
+ 'description': 'md5:ba12394c585ababea951cb6d2fcc6631',
+ 'timestamp': 1701302400,
+ 'upload_date': '20231130',
+ 'creator': 'Sue Li',
+ 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+ },
+ }, {
+ 'url': 'https://cooking.nytimes.com/recipes/1019516-overnight-oats',
+ 'md5': '2fe7965a3adc899913b8e25ada360823',
+ 'info_dict': {
+ 'id': '1019516',
+ 'ext': 'mp4',
+ 'timestamp': 1546387200,
+ 'description': 'md5:8856ce10239161bd2596ac335b9f9bfb',
+ 'upload_date': '20190102',
+ 'title': 'Overnight Oats',
+ 'creator': 'Genevieve Ko',
+ 'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
+ },
}]
def _real_extract(self, url):
page_id = self._match_id(url)
-
webpage = self._download_webpage(url, page_id)
+ recipe_data = self._search_nextjs_data(webpage, page_id)['props']['pageProps']['recipe']
- video_id = self._search_regex(
- r'data-video-id=["\'](\d+)', webpage, 'video id')
+ formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+ recipe_data['videoSrc'], page_id, 'mp4', m3u8_id='hls')
- return self._extract_video_from_id(video_id)
+ return {
+ **traverse_obj(recipe_data, {
+ 'id': ('id', {str_or_none}),
+ 'title': ('title', {str}),
+ 'description': ('topnote', {clean_html}),
+ 'timestamp': ('publishedAt', {int_or_none}),
+ 'creator': ('contentAttribution', 'cardByline', {str}),
+ }),
+ 'formats': formats,
+ 'subtitles': subtitles,
+ 'thumbnails': [{'url': thumb_url} for thumb_url in traverse_obj(
+ recipe_data, ('image', 'crops', 'recipe', ..., {url_or_none}))],
+ }
From acaf806c15f0a802ba286c23af02a10cf4bd4731 Mon Sep 17 00:00:00 2001
From: DmitryScaletta
Date: Mon, 5 Feb 2024 05:17:39 +0300
Subject: [PATCH 124/248] [ie/nuum] Add extractors (#8868)
Authored by: DmitryScaletta, seproDev
Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
yt_dlp/extractor/_extractors.py | 10 +-
yt_dlp/extractor/nuum.py | 199 ++++++++++++++++++++++++++++++++
yt_dlp/extractor/wasdtv.py | 159 -------------------------
3 files changed, 204 insertions(+), 164 deletions(-)
create mode 100644 yt_dlp/extractor/nuum.py
delete mode 100644 yt_dlp/extractor/wasdtv.py
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 36335286c..e7dd34c77 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1354,6 +1354,11 @@ from .nytimes import (
NYTimesCookingIE,
NYTimesCookingRecipeIE,
)
+from .nuum import (
+ NuumLiveIE,
+ NuumTabIE,
+ NuumMediaIE,
+)
from .nuvid import NuvidIE
from .nzherald import NZHeraldIE
from .nzonscreen import NZOnScreenIE
@@ -2315,11 +2320,6 @@ from .washingtonpost import (
WashingtonPostIE,
WashingtonPostArticleIE,
)
-from .wasdtv import (
- WASDTVStreamIE,
- WASDTVRecordIE,
- WASDTVClipIE,
-)
from .wat import WatIE
from .wdr import (
WDRIE,
diff --git a/yt_dlp/extractor/nuum.py b/yt_dlp/extractor/nuum.py
new file mode 100644
index 000000000..3db663ded
--- /dev/null
+++ b/yt_dlp/extractor/nuum.py
@@ -0,0 +1,199 @@
+import functools
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ OnDemandPagedList,
+ UserNotLive,
+ filter_dict,
+ int_or_none,
+ parse_iso8601,
+ str_or_none,
+ url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class NuumBaseIE(InfoExtractor):
+ def _call_api(self, path, video_id, description, query={}):
+ response = self._download_json(
+ f'https://nuum.ru/api/v2/{path}', video_id, query=query,
+ note=f'Downloading {description} metadata',
+ errnote=f'Unable to download {description} metadata')
+ if error := response.get('error'):
+ raise ExtractorError(f'API returned error: {error!r}')
+ return response['result']
+
+ def _get_channel_info(self, channel_name):
+ return self._call_api(
+ 'broadcasts/public', video_id=channel_name, description='channel',
+ query={
+ 'with_extra': 'true',
+ 'channel_name': channel_name,
+ 'with_deleted': 'true',
+ })
+
+ def _parse_video_data(self, container, extract_formats=True):
+ stream = traverse_obj(container, ('media_container_streams', 0, {dict})) or {}
+ media = traverse_obj(stream, ('stream_media', 0, {dict})) or {}
+ media_url = traverse_obj(media, (
+ 'media_meta', ('media_archive_url', 'media_url'), {url_or_none}), get_all=False)
+
+ video_id = str(container['media_container_id'])
+ is_live = media.get('media_status') == 'RUNNING'
+
+ formats, subtitles = None, None
+ if extract_formats:
+ formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+ media_url, video_id, 'mp4', live=is_live)
+
+ return filter_dict({
+ 'id': video_id,
+ 'is_live': is_live,
+ 'formats': formats,
+ 'subtitles': subtitles,
+ **traverse_obj(container, {
+ 'title': ('media_container_name', {str}),
+ 'description': ('media_container_description', {str}),
+ 'timestamp': ('created_at', {parse_iso8601}),
+ 'channel': ('media_container_channel', 'channel_name', {str}),
+ 'channel_id': ('media_container_channel', 'channel_id', {str_or_none}),
+ }),
+ **traverse_obj(stream, {
+ 'view_count': ('stream_total_viewers', {int_or_none}),
+ 'concurrent_view_count': ('stream_current_viewers', {int_or_none}),
+ }),
+ **traverse_obj(media, {
+ 'duration': ('media_duration', {int_or_none}),
+ 'thumbnail': ('media_meta', ('media_preview_archive_url', 'media_preview_url'), {url_or_none}),
+ }, get_all=False),
+ })
+
+
+class NuumMediaIE(NuumBaseIE):
+ IE_NAME = 'nuum:media'
+ _VALID_URL = r'https?://nuum\.ru/(?:streams|videos|clips)/(?P[\d]+)'
+ _TESTS = [{
+ 'url': 'https://nuum.ru/streams/1592713-7-days-to-die',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://nuum.ru/videos/1567547-toxi-hurtz',
+ 'md5': 'f1d9118a30403e32b702a204eb03aca3',
+ 'info_dict': {
+ 'id': '1567547',
+ 'ext': 'mp4',
+ 'title': 'Toxi$ - Hurtz',
+ 'description': '',
+ 'timestamp': 1702631651,
+ 'upload_date': '20231215',
+ 'thumbnail': r're:^https?://.+\.jpg',
+ 'view_count': int,
+ 'concurrent_view_count': int,
+ 'channel_id': '6911',
+ 'channel': 'toxis',
+ 'duration': 116,
+ },
+ }, {
+ 'url': 'https://nuum.ru/clips/1552564-pro-misu',
+ 'md5': 'b248ae1565b1e55433188f11beeb0ca1',
+ 'info_dict': {
+ 'id': '1552564',
+ 'ext': 'mp4',
+ 'title': 'Про Мису 🙃',
+ 'timestamp': 1701971828,
+ 'upload_date': '20231207',
+ 'thumbnail': r're:^https?://.+\.jpg',
+ 'view_count': int,
+ 'concurrent_view_count': int,
+ 'channel_id': '3320',
+ 'channel': 'Misalelik',
+ 'duration': 41,
+ },
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ video_data = self._call_api(f'media-containers/{video_id}', video_id, 'media')
+
+ return self._parse_video_data(video_data)
+
+
+class NuumLiveIE(NuumBaseIE):
+ IE_NAME = 'nuum:live'
+ _VALID_URL = r'https?://nuum\.ru/channel/(?P[^/#?]+)/?(?:$|[#?])'
+ _TESTS = [{
+ 'url': 'https://nuum.ru/channel/mts_live',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ channel = self._match_id(url)
+ channel_info = self._get_channel_info(channel)
+ if traverse_obj(channel_info, ('channel', 'channel_is_live')) is False:
+ raise UserNotLive(video_id=channel)
+
+ info = self._parse_video_data(channel_info['media_container'])
+ return {
+ 'webpage_url': f'https://nuum.ru/streams/{info["id"]}',
+ 'extractor_key': NuumMediaIE.ie_key(),
+ 'extractor': NuumMediaIE.IE_NAME,
+ **info,
+ }
+
+
+class NuumTabIE(NuumBaseIE):
+ IE_NAME = 'nuum:tab'
+ _VALID_URL = r'https?://nuum\.ru/channel/(?P[^/#?]+)/(?Pstreams|videos|clips)'
+ _TESTS = [{
+ 'url': 'https://nuum.ru/channel/dankon_/clips',
+ 'info_dict': {
+ 'id': 'dankon__clips',
+ 'title': 'Dankon_',
+ },
+ 'playlist_mincount': 29,
+ }, {
+ 'url': 'https://nuum.ru/channel/dankon_/videos',
+ 'info_dict': {
+ 'id': 'dankon__videos',
+ 'title': 'Dankon_',
+ },
+ 'playlist_mincount': 2,
+ }, {
+ 'url': 'https://nuum.ru/channel/dankon_/streams',
+ 'info_dict': {
+ 'id': 'dankon__streams',
+ 'title': 'Dankon_',
+ },
+ 'playlist_mincount': 1,
+ }]
+
+ _PAGE_SIZE = 50
+
+ def _fetch_page(self, channel_id, tab_type, tab_id, page):
+ CONTAINER_TYPES = {
+ 'clips': ['SHORT_VIDEO', 'REVIEW_VIDEO'],
+ 'videos': ['LONG_VIDEO'],
+ 'streams': ['SINGLE'],
+ }
+
+ media_containers = self._call_api(
+ 'media-containers', video_id=tab_id, description=f'{tab_type} tab page {page + 1}',
+ query={
+ 'limit': self._PAGE_SIZE,
+ 'offset': page * self._PAGE_SIZE,
+ 'channel_id': channel_id,
+ 'media_container_status': 'STOPPED',
+ 'media_container_type': CONTAINER_TYPES[tab_type],
+ })
+ for container in traverse_obj(media_containers, (..., {dict})):
+ metadata = self._parse_video_data(container, extract_formats=False)
+ yield self.url_result(f'https://nuum.ru/videos/{metadata["id"]}', NuumMediaIE, **metadata)
+
+ def _real_extract(self, url):
+ channel_name, tab_type = self._match_valid_url(url).group('id', 'type')
+ tab_id = f'{channel_name}_{tab_type}'
+ channel_data = self._get_channel_info(channel_name)['channel']
+
+ return self.playlist_result(OnDemandPagedList(functools.partial(
+ self._fetch_page, channel_data['channel_id'], tab_type, tab_id), self._PAGE_SIZE),
+ playlist_id=tab_id, playlist_title=channel_data.get('channel_name'))
diff --git a/yt_dlp/extractor/wasdtv.py b/yt_dlp/extractor/wasdtv.py
deleted file mode 100644
index f57c619b5..000000000
--- a/yt_dlp/extractor/wasdtv.py
+++ /dev/null
@@ -1,159 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
- int_or_none,
- parse_iso8601,
- traverse_obj,
- try_get,
-)
-
-
-class WASDTVBaseIE(InfoExtractor):
-
- def _fetch(self, path, video_id, description, query={}):
- response = self._download_json(
- f'https://wasd.tv/api/{path}', video_id, query=query,
- note=f'Downloading {description} metadata',
- errnote=f'Unable to download {description} metadata')
- error = response.get('error')
- if error:
- raise ExtractorError(f'{self.IE_NAME} returned error: {error}', expected=True)
- return response.get('result')
-
- def _extract_thumbnails(self, thumbnails_dict):
- return [{
- 'url': url,
- 'preference': index,
- } for index, url in enumerate(
- traverse_obj(thumbnails_dict, (('small', 'medium', 'large'),))) if url]
-
- def _real_extract(self, url):
- container = self._get_container(url)
- stream = traverse_obj(container, ('media_container_streams', 0))
- media = try_get(stream, lambda x: x['stream_media'][0])
- if not media:
- raise ExtractorError('Can not extract media data.', expected=True)
- media_meta = media.get('media_meta')
- media_url, is_live = self._get_media_url(media_meta)
- video_id = media.get('media_id') or container.get('media_container_id')
- formats, subtitles = self._extract_m3u8_formats_and_subtitles(media_url, video_id, 'mp4')
- return {
- 'id': str(video_id),
- 'title': container.get('media_container_name') or self._og_search_title(self._download_webpage(url, video_id)),
- 'description': container.get('media_container_description'),
- 'thumbnails': self._extract_thumbnails(media_meta.get('media_preview_images')),
- 'timestamp': parse_iso8601(container.get('created_at')),
- 'view_count': int_or_none(stream.get('stream_current_viewers' if is_live else 'stream_total_viewers')),
- 'is_live': is_live,
- 'formats': formats,
- 'subtitles': subtitles,
- }
-
- def _get_container(self, url):
- raise NotImplementedError('Subclass for get media container')
-
- def _get_media_url(self, media_meta):
- raise NotImplementedError('Subclass for get media url')
-
-
-class WASDTVStreamIE(WASDTVBaseIE):
- IE_NAME = 'wasdtv:stream'
- _VALID_URL = r'https?://wasd\.tv/(?P[^/#?]+)$'
- _TESTS = [{
- 'url': 'https://wasd.tv/24_7',
- 'info_dict': {
- 'id': '559738',
- 'ext': 'mp4',
- 'title': 'Live 24/7 Music',
- 'description': '24/7 Music',
- 'timestamp': int,
- 'upload_date': r're:^\d{8}$',
- 'is_live': True,
- 'view_count': int,
- },
- }]
-
- def _get_container(self, url):
- nickname = self._match_id(url)
- channel = self._fetch(f'channels/nicknames/{nickname}', video_id=nickname, description='channel')
- channel_id = channel.get('channel_id')
- containers = self._fetch(
- 'v2/media-containers', channel_id, 'running media containers',
- query={
- 'channel_id': channel_id,
- 'media_container_type': 'SINGLE',
- 'media_container_status': 'RUNNING',
- })
- if not containers:
- raise ExtractorError(f'{nickname} is offline', expected=True)
- return containers[0]
-
- def _get_media_url(self, media_meta):
- return media_meta['media_url'], True
-
-
-class WASDTVRecordIE(WASDTVBaseIE):
- IE_NAME = 'wasdtv:record'
- _VALID_URL = r'https?://wasd\.tv/[^/#?]+(?:/videos)?\?record=(?P\d+)$'
- _TESTS = [{
- 'url': 'https://wasd.tv/spacemita/videos?record=907755',
- 'md5': 'c9899dd85be4cc997816ff9f9ca516ce',
- 'info_dict': {
- 'id': '906825',
- 'ext': 'mp4',
- 'title': 'Музыкальный',
- 'description': 'md5:f510388d929ff60ae61d4c3cab3137cc',
- 'timestamp': 1645812079,
- 'upload_date': '20220225',
- 'thumbnail': r're:^https?://.+\.jpg',
- 'is_live': False,
- 'view_count': int,
- },
- }, {
- 'url': 'https://wasd.tv/spacemita?record=907755',
- 'only_matching': True,
- }]
-
- def _get_container(self, url):
- container_id = self._match_id(url)
- return self._fetch(
- f'v2/media-containers/{container_id}', container_id, 'media container')
-
- def _get_media_url(self, media_meta):
- media_archive_url = media_meta.get('media_archive_url')
- if media_archive_url:
- return media_archive_url, False
- return media_meta['media_url'], True
-
-
-class WASDTVClipIE(WASDTVBaseIE):
- IE_NAME = 'wasdtv:clip'
- _VALID_URL = r'https?://wasd\.tv/[^/#?]+/clips\?clip=(?P\d+)$'
- _TESTS = [{
- 'url': 'https://wasd.tv/spacemita/clips?clip=26804',
- 'md5': '818885e720143d7a4e776ff66fcff148',
- 'info_dict': {
- 'id': '26804',
- 'ext': 'mp4',
- 'title': 'Пуш флексит на голове стримера',
- 'timestamp': 1646682908,
- 'upload_date': '20220307',
- 'thumbnail': r're:^https?://.+\.jpg',
- 'view_count': int,
- },
- }]
-
- def _real_extract(self, url):
- clip_id = self._match_id(url)
- clip = self._fetch(f'v2/clips/{clip_id}', video_id=clip_id, description='clip')
- clip_data = clip.get('clip_data')
- formats, subtitles = self._extract_m3u8_formats_and_subtitles(clip_data.get('url'), video_id=clip_id, ext='mp4')
- return {
- 'id': clip_id,
- 'title': clip.get('clip_title') or self._og_search_title(self._download_webpage(url, clip_id, fatal=False)),
- 'thumbnails': self._extract_thumbnails(clip_data.get('preview')),
- 'timestamp': parse_iso8601(clip.get('created_at')),
- 'view_count': int_or_none(clip.get('clip_views_count')),
- 'formats': formats,
- 'subtitles': subtitles,
- }
From 35d96982f1033e36215d323317981ee17e8ab0d5 Mon Sep 17 00:00:00 2001
From: Chocobozzz
Date: Mon, 5 Feb 2024 20:58:32 +0100
Subject: [PATCH 125/248] [ie/peertube] Update instances (#9070)
Authored by: Chocobozzz
---
yt_dlp/extractor/peertube.py | 972 ++++++++++++++++++++++-------------
1 file changed, 610 insertions(+), 362 deletions(-)
diff --git a/yt_dlp/extractor/peertube.py b/yt_dlp/extractor/peertube.py
index 68e15737b..730b2393e 100644
--- a/yt_dlp/extractor/peertube.py
+++ b/yt_dlp/extractor/peertube.py
@@ -19,636 +19,902 @@ from ..utils import (
class PeerTubeIE(InfoExtractor):
_INSTANCES_RE = r'''(?:
# Taken from https://instances.joinpeertube.org/instances
- 40two\.tube|
- a\.metube\.ch|
- advtv\.ml|
- algorithmic\.tv|
- alimulama\.com|
- arcana\.fun|
- archive\.vidicon\.org|
- artefac-paris\.tv|
- auf1\.eu|
+ 0ch\.tv|
+ 3dctube\.3dcandy\.social|
+ all\.electric\.kitchen|
+ alterscope\.fr|
+ anarchy\.tube|
+ apathy\.tv|
+ apertatube\.net|
+ archive\.nocopyrightintended\.tv|
+ archive\.reclaim\.tv|
+ area51\.media|
+ astrotube-ufe\.obspm\.fr|
+ astrotube\.obspm\.fr|
+ audio\.freediverse\.com|
+ azxtube\.youssefc\.tn|
+ bark\.video|
battlepenguin\.video|
- beertube\.epgn\.ch|
- befree\.nohost\.me|
+ bava\.tv|
+ bee-tube\.fr|
+ beetoons\.tv|
+ biblion\.refchat\.net|
+ biblioteca\.theowlclub\.net|
bideoak\.argia\.eus|
- birkeundnymphe\.de|
+ bideoteka\.eus|
+ birdtu\.be|
bitcointv\.com|
- cattube\.org|
- clap\.nerv-project\.eu|
- climatejustice\.video|
+ bonn\.video|
+ breeze\.tube|
+ brioco\.live|
+ brocosoup\.fr|
+ canal\.facil\.services|
+ canard\.tube|
+ cdn01\.tilvids\.com|
+ celluloid-media\.huma-num\.fr|
+ chicago1\.peertube\.support|
+ cliptube\.org|
+ cloudtube\.ise\.fraunhofer\.de|
comf\.tube|
+ comics\.peertube\.biz|
+ commons\.tube|
+ communitymedia\.video|
conspiracydistillery\.com|
+ crank\.recoil\.org|
+ dalek\.zone|
+ dalliance\.network|
+ dangly\.parts|
darkvapor\.nohost\.me|
daschauher\.aksel\.rocks|
digitalcourage\.video|
- dreiecksnebel\.alex-detsch\.de|
- eduvid\.org|
+ displayeurope\.video|
+ ds106\.tv|
+ dud-video\.inf\.tu-dresden\.de|
+ dud175\.inf\.tu-dresden\.de|
+ dytube\.com|
+ ebildungslabor\.video|
evangelisch\.video|
- exo\.tube|
fair\.tube|
+ fedi\.video|
+ fedimovie\.com|
fediverse\.tv|
film\.k-prod\.fr|
- flim\.txmn\.tk|
+ flipboard\.video|
+ foss\.video|
+ fossfarmers\.company|
fotogramas\.politicaconciencia\.org|
- ftsi\.ru|
- gary\.vger\.cloud|
- graeber\.video|
+ freediverse\.com|
+ freesoto-u2151\.vm\.elestio\.app|
+ freesoto\.tv|
+ garr\.tv|
greatview\.video|
grypstube\.uni-greifswald\.de|
- highvoltage\.tv|
- hpstube\.fr|
- htp\.live|
- hyperreal\.tube|
+ habratube\.site|
+ ilbjach\.ru|
+ infothema\.net|
+ itvplus\.iiens\.net|
+ johnydeep\.net|
juggling\.digital|
+ jupiter\.tube|
+ kadras\.live|
kino\.kompot\.si|
kino\.schuerz\.at|
kinowolnosc\.pl|
kirche\.peertube-host\.de|
+ kiwi\.froggirl\.club|
kodcast\.com|
kolektiva\.media|
- kraut\.zone|
+ kpop\.22x22\.ru|
kumi\.tube|
+ la2\.peertube\.support|
+ la3\.peertube\.support|
+ la4\.peertube\.support|
lastbreach\.tv|
- lepetitmayennais\.fr\.nf|
- lexx\.impa\.me|
- libertynode\.tv|
- libra\.syntazia\.org|
- libremedia\.video|
+ lawsplaining\.peertube\.biz|
+ leopard\.tube|
+ live\.codinglab\.ch|
live\.libratoi\.org|
- live\.nanao\.moe|
- live\.toobnix\.org|
- livegram\.net|
- lolitube\.freedomchan\.moe|
+ live\.oldskool\.fi|
+ live\.solari\.com|
lucarne\.balsamine\.be|
- maindreieck-tv\.de|
- mani\.tube|
- manicphase\.me|
+ luxtube\.lu|
+ makertube\.net|
+ media\.econoalchemist\.com|
+ media\.exo\.cat|
media\.fsfe\.org|
media\.gzevd\.de|
- media\.inno3\.cricket|
- media\.kaitaia\.life|
+ media\.interior\.edu\.uy|
media\.krashboyz\.org|
- media\.over-world\.org|
- media\.skewed\.de|
+ media\.mzhd\.de|
+ media\.smz-ma\.de|
+ media\.theplattform\.net|
media\.undeadnetwork\.de|
+ medias\.debrouillonet\.org|
medias\.pingbase\.net|
+ mediatube\.fermalo\.fr|
melsungen\.peertube-host\.de|
- mirametube\.fr|
- mojotube\.net|
- monplaisirtube\.ddns\.net|
+ merci-la-police\.fr|
+ mindlyvideos\.com|
+ mirror\.peertube\.metalbanana\.net|
+ mirrored\.rocks|
+ mix\.video|
mountaintown\.video|
- my\.bunny\.cafe|
- myfreetube\.de|
+ movies\.metricsmaster\.eu|
+ mtube\.mooo\.com|
mytube\.kn-cloud\.de|
+ mytube\.le5emeaxe\.fr|
mytube\.madzel\.de|
- myworkoutarenapeertube\.cf|
+ nadajemy\.com|
nanawel-peertube\.dyndns\.org|
- nastub\.cz|
- offenes\.tv|
- orgdup\.media|
- ovaltube\.codinglab\.ch|
+ neat\.tube|
+ nethack\.tv|
+ nicecrew\.tv|
+ nightshift\.minnix\.dev|
+ nolog\.media|
+ nyltube\.nylarea\.com|
+ ocfedtest\.hosted\.spacebear\.ee|
+ openmedia\.edunova\.it|
p2ptv\.ru|
p\.eertu\.be|
p\.lu|
+ pastafriday\.club|
+ patriottube\.sonsofliberty\.red|
+ pcbu\.nl|
peer\.azurs\.fr|
- peertube1\.zeteo\.me|
+ peer\.d0g4\.me|
+ peer\.lukeog\.com|
+ peer\.madiator\.cloud|
+ peer\.raise-uav\.com|
+ peershare\.togart\.de|
+ peertube-blablalinux\.be|
+ peertube-demo\.learning-hub\.fr|
+ peertube-docker\.cpy\.re|
+ peertube-eu\.howlround\.com|
+ peertube-u5014\.vm\.elestio\.app|
+ peertube-us\.howlround\.com|
peertube\.020\.pl|
peertube\.0x5e\.eu|
+ peertube\.1984\.cz|
+ peertube\.2i2l\.net|
+ peertube\.adjutor\.xyz|
+ peertube\.adresse\.data\.gouv\.fr|
peertube\.alpharius\.io|
peertube\.am-networks\.fr|
peertube\.anduin\.net|
- peertube\.anzui\.dev|
- peertube\.arbleizez\.bzh|
+ peertube\.anti-logic\.com|
+ peertube\.arch-linux\.cz|
peertube\.art3mis\.de|
- peertube\.atilla\.org|
+ peertube\.artsrn\.ualberta\.ca|
+ peertube\.askan\.info|
+ peertube\.astral0pitek\.synology\.me|
peertube\.atsuchan\.page|
- peertube\.aukfood\.net|
- peertube\.aventer\.biz|
+ peertube\.automat\.click|
peertube\.b38\.rural-it\.org|
- peertube\.beeldengeluid\.nl|
peertube\.be|
+ peertube\.beeldengeluid\.nl|
peertube\.bgzashtita\.es|
- peertube\.bitsandlinux\.com|
+ peertube\.bike|
+ peertube\.bildung-ekhn\.de|
peertube\.biz|
- peertube\.boba\.best|
peertube\.br0\.fr|
peertube\.bridaahost\.ynh\.fr|
peertube\.bubbletea\.dev|
peertube\.bubuit\.net|
peertube\.cabaal\.net|
- peertube\.cats-home\.net|
- peertube\.chemnitz\.freifunk\.net|
- peertube\.chevro\.fr|
- peertube\.chrisspiegl\.com|
+ peertube\.chatinbit\.com|
+ peertube\.chaunchy\.com|
+ peertube\.chir\.rs|
+ peertube\.christianpacaud\.com|
peertube\.chtisurel\.net|
+ peertube\.chuggybumba\.com|
peertube\.cipherbliss\.com|
+ peertube\.cirkau\.art|
+ peertube\.cloud\.nerdraum\.de|
peertube\.cloud\.sans\.pub|
+ peertube\.coko\.foundation|
+ peertube\.communecter\.org|
+ peertube\.concordia\.social|
+ peertube\.corrigan\.xyz|
peertube\.cpge-brizeux\.fr|
peertube\.ctseuro\.com|
peertube\.cuatrolibertades\.org|
- peertube\.cybercirujas\.club|
- peertube\.cythin\.com|
+ peertube\.cube4fun\.net|
+ peertube\.dair-institute\.org|
peertube\.davigge\.com|
peertube\.dc\.pini\.fr|
+ peertube\.deadtom\.me|
peertube\.debian\.social|
+ peertube\.delta0189\.xyz|
peertube\.demonix\.fr|
peertube\.designersethiques\.org|
peertube\.desmu\.fr|
- peertube\.devloprog\.org|
peertube\.devol\.it|
- peertube\.dtmf\.ca|
- peertube\.ecologie\.bzh|
+ peertube\.dk|
+ peertube\.doesstuff\.social|
+ peertube\.eb8\.org|
+ peertube\.education-forum\.com|
+ peertube\.elforcer\.ru|
+ peertube\.em\.id\.lv|
+ peertube\.ethibox\.fr|
peertube\.eu\.org|
peertube\.european-pirates\.eu|
+ peertube\.eus|
peertube\.euskarabildua\.eus|
+ peertube\.expi\.studio|
+ peertube\.familie-berner\.de|
+ peertube\.familleboisteau\.fr|
+ peertube\.fedihost\.website|
peertube\.fenarinarsa\.com|
- peertube\.fomin\.site|
- peertube\.forsud\.be|
- peertube\.francoispelletier\.org|
- peertube\.freenet\.ru|
- peertube\.freetalklive\.com|
+ peertube\.festnoz\.de|
+ peertube\.forteza\.fr|
+ peertube\.freestorm\.online|
peertube\.functional\.cafe|
- peertube\.gardeludwig\.fr|
+ peertube\.gaminglinux\.fr|
peertube\.gargantia\.fr|
- peertube\.gcfamily\.fr|
+ peertube\.geekgalaxy\.fr|
+ peertube\.gemlog\.ca|
peertube\.genma\.fr|
peertube\.get-racing\.de|
+ peertube\.ghis94\.ovh|
peertube\.gidikroon\.eu|
- peertube\.gruezishop\.ch|
- peertube\.habets\.house|
- peertube\.hackerfraternity\.org|
+ peertube\.giftedmc\.com|
+ peertube\.grosist\.fr|
+ peertube\.gruntwerk\.org|
+ peertube\.gsugambit\.com|
+ peertube\.hackerfoo\.com|
+ peertube\.hellsite\.net|
+ peertube\.helvetet\.eu|
+ peertube\.histoirescrepues\.fr|
+ peertube\.home\.x0r\.fr|
+ peertube\.hyperfreedom\.org|
peertube\.ichigo\.everydayimshuflin\.com|
- peertube\.ignifi\.me|
+ peertube\.ifwo\.eu|
+ peertube\.in\.ua|
peertube\.inapurna\.org|
peertube\.informaction\.info|
peertube\.interhop\.org|
- peertube\.iselfhost\.com|
peertube\.it|
+ peertube\.it-arts\.net|
peertube\.jensdiemer\.de|
- peertube\.joffreyverd\.fr|
+ peertube\.johntheserg\.al|
+ peertube\.kaleidos\.net|
peertube\.kalua\.im|
- peertube\.kathryl\.fr|
+ peertube\.kcore\.org|
peertube\.keazilla\.net|
peertube\.klaewyss\.fr|
- peertube\.kodcast\.com|
+ peertube\.kleph\.eu|
+ peertube\.kodein\.be|
+ peertube\.kooperatywa\.tech|
+ peertube\.kriom\.net|
peertube\.kx\.studio|
+ peertube\.kyriog\.eu|
+ peertube\.la-famille-muller\.fr|
+ peertube\.labeuropereunion\.eu|
peertube\.lagvoid\.com|
- peertube\.lavallee\.tech|
- peertube\.le5emeaxe\.fr|
- peertube\.lestutosdeprocessus\.fr|
- peertube\.librenet\.co\.za|
+ peertube\.lhc\.net\.br|
+ peertube\.libresolutions\.network|
+ peertube\.libretic\.fr|
+ peertube\.librosphere\.fr|
peertube\.logilab\.fr|
+ peertube\.lon\.tv|
peertube\.louisematic\.site|
peertube\.luckow\.org|
peertube\.luga\.at|
peertube\.lyceeconnecte\.fr|
- peertube\.manalejandro\.com|
+ peertube\.madixam\.xyz|
+ peertube\.magicstone\.dev|
+ peertube\.marienschule\.de|
peertube\.marud\.fr|
- peertube\.mattone\.net|
peertube\.maxweiss\.io|
+ peertube\.miguelcr\.me|
+ peertube\.mikemestnik\.net|
+ peertube\.mobilsicher\.de|
peertube\.monlycee\.net|
peertube\.mxinfo\.fr|
- peertube\.myrasp\.eu|
- peertube\.nebelcloud\.de|
+ peertube\.naln1\.ca|
peertube\.netzbegruenung\.de|
- peertube\.newsocial\.tech|
peertube\.nicolastissot\.fr|
+ peertube\.nogafam\.fr|
+ peertube\.normalgamingcommunity\.cz|
peertube\.nz|
peertube\.offerman\.com|
+ peertube\.ohioskates\.com|
+ peertube\.onionstorm\.net|
peertube\.opencloud\.lu|
- peertube\.orthus\.link|
- peertube\.patapouf\.xyz|
- peertube\.pi2\.dev|
- peertube\.plataformess\.org|
- peertube\.pl|
- peertube\.portaesgnos\.org|
+ peertube\.otakufarms\.com|
+ peertube\.paladyn\.org|
+ peertube\.pix-n-chill\.fr|
peertube\.r2\.enst\.fr|
peertube\.r5c3\.fr|
- peertube\.radres\.xyz|
- peertube\.red|
- peertube\.robonomics\.network|
- peertube\.rtnkv\.cloud|
- peertube\.runfox\.tk|
+ peertube\.redpill-insight\.com|
+ peertube\.researchinstitute\.at|
+ peertube\.revelin\.fr|
+ peertube\.rlp\.schule|
+ peertube\.rokugan\.fr|
+ peertube\.rougevertbleu\.tv|
+ peertube\.roundpond\.net|
+ peertube\.rural-it\.org|
peertube\.satoshishop\.de|
- peertube\.scic-tetris\.org|
+ peertube\.scyldings\.com|
peertube\.securitymadein\.lu|
+ peertube\.semperpax\.com|
peertube\.semweb\.pro|
- peertube\.social\.my-wan\.de|
- peertube\.soykaf\.org|
- peertube\.stefofficiel\.me|
+ peertube\.sensin\.eu|
+ peertube\.sidh\.bzh|
+ peertube\.skorpil\.cz|
+ peertube\.smertrios\.com|
+ peertube\.sqweeb\.net|
+ peertube\.stattzeitung\.org|
peertube\.stream|
peertube\.su|
peertube\.swrs\.net|
peertube\.takeko\.cyou|
- peertube\.tangentfox\.com|
peertube\.taxinachtegel\.de|
- peertube\.thenewoil\.xyz|
+ peertube\.teftera\.com|
+ peertube\.teutronic-services\.de|
peertube\.ti-fr\.com|
peertube\.tiennot\.net|
- peertube\.troback\.com|
+ peertube\.tmp\.rcp\.tf|
peertube\.tspu\.edu\.ru|
- peertube\.tux\.ovh|
peertube\.tv|
peertube\.tweb\.tv|
- peertube\.ucy\.de|
peertube\.underworld\.fr|
- peertube\.us\.to|
- peertube\.ventresmous\.fr|
+ peertube\.vapronva\.pw|
+ peertube\.veen\.world|
+ peertube\.vesdia\.eu|
+ peertube\.virtual-assembly\.org|
+ peertube\.viviers-fibre\.net|
peertube\.vlaki\.cz|
- peertube\.w\.utnw\.de|
- peertube\.westring\.digital|
+ peertube\.wiesbaden\.social|
+ peertube\.wivodaim\.net|
+ peertube\.wtf|
+ peertube\.wtfayla\.net|
+ peertube\.xrcb\.cat|
peertube\.xwiki\.com|
+ peertube\.zd\.do|
+ peertube\.zetamc\.net|
+ peertube\.zmuuf\.org|
peertube\.zoz-serv\.org|
+ peertube\.zwindler\.fr|
peervideo\.ru|
periscope\.numenaute\.org|
- perron-tube\.de|
+ pete\.warpnine\.de|
petitlutinartube\.fr|
phijkchu\.com|
- pierre\.tube|
+ phoenixproject\.group|
piraten\.space|
- play\.rosano\.ca|
+ pirtube\.calut\.fr|
+ pityu\.flaki\.hu|
+ play\.mittdata\.se|
player\.ojamajo\.moe|
- plextube\.nl|
- pocketnetpeertube1\.nohost\.me|
- pocketnetpeertube3\.nohost\.me|
- pocketnetpeertube4\.nohost\.me|
- pocketnetpeertube5\.nohost\.me|
- pocketnetpeertube6\.nohost\.me|
- pt\.24-7\.ro|
- pt\.apathy\.top|
+ podlibre\.video|
+ portal\.digilab\.nfa\.cz|
+ private\.fedimovie\.com|
+ pt01\.lehrerfortbildung-bw\.de|
pt\.diaspodon\.fr|
- pt\.fedi\.tech|
- pt\.maciej\.website|
+ pt\.freedomwolf\.cc|
+ pt\.gordons\.gen\.nz|
+ pt\.ilyamikcoder\.com|
+ pt\.irnok\.net|
+ pt\.mezzo\.moe|
+ pt\.na4\.eu|
+ pt\.netcraft\.ch|
+ pt\.rwx\.ch|
+ pt\.sfunk1x\.com|
+ pt\.thishorsie\.rocks|
+ pt\.vern\.cc|
ptb\.lunarviews\.net|
- ptmir1\.inter21\.net|
- ptmir2\.inter21\.net|
- ptmir3\.inter21\.net|
- ptmir4\.inter21\.net|
- ptmir5\.inter21\.net|
- ptube\.horsentiers\.fr|
- ptube\.xmanifesto\.club|
- queermotion\.org|
- re-wizja\.re-medium\.com|
- regarder\.sans\.pub|
- ruraletv\.ovh|
- s1\.gegenstimme\.tv|
- s2\.veezee\.tube|
+ ptube\.de|
+ ptube\.ranranhome\.info|
+ puffy\.tube|
+ puppet\.zone|
+ qtube\.qlyoung\.net|
+ quantube\.win|
+ rankett\.net|
+ replay\.jres\.org|
+ review\.peertube\.biz|
sdmtube\.fr|
- sender-fm\.veezee\.tube|
- serv1\.wiki-tube\.de|
+ secure\.direct-live\.net|
+ secure\.scanovid\.com|
+ seka\.pona\.la|
serv3\.wiki-tube\.de|
- sickstream\.net|
- sleepy\.tube|
+ skeptube\.fr|
+ social\.fedimovie\.com|
+ socpeertube\.ru|
sovran\.video|
+ special\.videovortex\.tv|
spectra\.video|
+ stl1988\.peertube-host\.de|
+ stream\.biovisata\.lt|
+ stream\.conesphere\.cloud|
stream\.elven\.pw|
+ stream\.jurnalfm\.md|
stream\.k-prod\.fr|
- stream\.shahab\.nohost\.me|
- streamsource\.video|
+ stream\.litera\.tools|
+ stream\.nuemedia\.se|
+ stream\.rlp-media\.de|
+ stream\.vrse\.be|
studios\.racer159\.com|
- testtube\.florimond\.eu|
+ styxhexenhammer666\.com|
+ syrteplay\.obspm\.fr|
+ t\.0x0\.st|
+ tbh\.co-shaoghal\.net|
+ test-fab\.ynh\.fr|
+ testube\.distrilab\.fr|
tgi\.hosted\.spacebear\.ee|
- thaitube\.in\.th|
- the\.jokertv\.eu|
theater\.ethernia\.net|
thecool\.tube|
+ thevideoverse\.com|
tilvids\.com|
- toob\.bub\.org|
- tpaw\.video|
- truetube\.media|
- tuba\.lhub\.pl|
- tube-aix-marseille\.beta\.education\.fr|
- tube-amiens\.beta\.education\.fr|
- tube-besancon\.beta\.education\.fr|
- tube-bordeaux\.beta\.education\.fr|
- tube-clermont-ferrand\.beta\.education\.fr|
- tube-corse\.beta\.education\.fr|
- tube-creteil\.beta\.education\.fr|
- tube-dijon\.beta\.education\.fr|
- tube-education\.beta\.education\.fr|
- tube-grenoble\.beta\.education\.fr|
- tube-lille\.beta\.education\.fr|
- tube-limoges\.beta\.education\.fr|
- tube-montpellier\.beta\.education\.fr|
- tube-nancy\.beta\.education\.fr|
- tube-nantes\.beta\.education\.fr|
- tube-nice\.beta\.education\.fr|
- tube-normandie\.beta\.education\.fr|
- tube-orleans-tours\.beta\.education\.fr|
- tube-outremer\.beta\.education\.fr|
- tube-paris\.beta\.education\.fr|
- tube-poitiers\.beta\.education\.fr|
- tube-reims\.beta\.education\.fr|
- tube-rennes\.beta\.education\.fr|
- tube-strasbourg\.beta\.education\.fr|
- tube-toulouse\.beta\.education\.fr|
- tube-versailles\.beta\.education\.fr|
- tube1\.it\.tuwien\.ac\.at|
+ tinkerbetter\.tube|
+ tinsley\.video|
+ trailers\.ddigest\.com|
+ tube-action-educative\.apps\.education\.fr|
+ tube-arts-lettres-sciences-humaines\.apps\.education\.fr|
+ tube-cycle-2\.apps\.education\.fr|
+ tube-cycle-3\.apps\.education\.fr|
+ tube-education-physique-et-sportive\.apps\.education\.fr|
+ tube-enseignement-professionnel\.apps\.education\.fr|
+ tube-institutionnel\.apps\.education\.fr|
+ tube-langues-vivantes\.apps\.education\.fr|
+ tube-maternelle\.apps\.education\.fr|
+ tube-numerique-educatif\.apps\.education\.fr|
+ tube-sciences-technologies\.apps\.education\.fr|
+ tube-test\.apps\.education\.fr|
+ tube1\.perron-service\.de|
+ tube\.9minuti\.it|
tube\.abolivier\.bzh|
- tube\.ac-amiens\.fr|
- tube\.aerztefueraufklaerung\.de|
- tube\.alexx\.ml|
+ tube\.alado\.space|
tube\.amic37\.fr|
- tube\.anufrij\.de|
- tube\.apolut\.net|
- tube\.arkhalabs\.io|
+ tube\.area404\.cloud|
tube\.arthack\.nz|
- tube\.as211696\.net|
- tube\.avensio\.de|
+ tube\.asulia\.fr|
+ tube\.awkward\.company|
tube\.azbyka\.ru|
tube\.azkware\.net|
- tube\.bachaner\.fr|
- tube\.bmesh\.org|
- tube\.borked\.host|
+ tube\.bartrip\.me\.uk|
+ tube\.belowtoxic\.media|
+ tube\.bingle\.plus|
+ tube\.bit-friends\.de|
tube\.bstly\.de|
- tube\.chaoszone\.tv|
- tube\.chatelet\.ovh|
- tube\.cloud-libre\.eu|
+ tube\.chosto\.me|
tube\.cms\.garden|
- tube\.cowfee\.moe|
- tube\.cryptography\.dog|
- tube\.darknight-coffee\.org|
- tube\.dev\.lhub\.pl|
+ tube\.communia\.org|
+ tube\.cyberia\.club|
+ tube\.cybershock\.life|
+ tube\.dembased\.xyz|
+ tube\.dev\.displ\.eu|
+ tube\.digitalesozialearbeit\.de|
tube\.distrilab\.fr|
+ tube\.doortofreedom\.org|
tube\.dsocialize\.net|
+ tube\.e-jeremy\.com|
tube\.ebin\.club|
+ tube\.elemac\.fr|
+ tube\.erzbistum-hamburg\.de|
+ tube\.exozy\.me|
tube\.fdn\.fr|
- tube\.florimond\.eu|
- tube\.foxarmy\.ml|
- tube\.foxden\.party|
- tube\.frischesicht\.de|
+ tube\.fedi\.quebec|
+ tube\.fediverse\.at|
+ tube\.felinn\.org|
+ tube\.flokinet\.is|
+ tube\.foad\.me\.uk|
+ tube\.freepeople\.fr|
+ tube\.friloux\.me|
+ tube\.froth\.zone|
+ tube\.fulda\.social|
tube\.futuretic\.fr|
- tube\.gnous\.eu|
+ tube\.g1zm0\.de|
+ tube\.g4rf\.net|
+ tube\.gaiac\.io|
+ tube\.geekyboo\.net|
+ tube\.genb\.de|
+ tube\.ghk-academy\.info|
+ tube\.gi-it\.de|
tube\.grap\.coop|
tube\.graz\.social|
tube\.grin\.hu|
- tube\.hackerscop\.org|
- tube\.hordearii\.fr|
+ tube\.hokai\.lol|
+ tube\.int5\.net|
+ tube\.interhacker\.space|
+ tube\.invisible\.ch|
+ tube\.io18\.top|
+ tube\.itsg\.host|
tube\.jeena\.net|
- tube\.kai-stuht\.com|
+ tube\.kh-berlin\.de|
tube\.kockatoo\.org|
tube\.kotur\.org|
+ tube\.koweb\.fr|
+ tube\.la-dina\.net|
+ tube\.lab\.nrw|
tube\.lacaveatonton\.ovh|
+ tube\.laurent-malys\.fr|
+ tube\.leetdreams\.ch|
tube\.linkse\.media|
tube\.lokad\.com|
tube\.lucie-philou\.com|
- tube\.melonbread\.xyz|
- tube\.mfraters\.net|
- tube\.motuhake\.xyz|
- tube\.mrbesen\.de|
- tube\.nah\.re|
- tube\.nchoco\.net|
+ tube\.media-techport\.de|
+ tube\.morozoff\.pro|
+ tube\.neshweb\.net|
+ tube\.nestor\.coop|
+ tube\.network\.europa\.eu|
+ tube\.nicfab\.eu|
+ tube\.nieuwwestbrabant\.nl|
+ tube\.nogafa\.org|
tube\.novg\.net|
tube\.nox-rhea\.org|
tube\.nuagelibre\.fr|
+ tube\.numerique\.gouv\.fr|
+ tube\.nuxnik\.com|
tube\.nx12\.net|
tube\.octaplex\.net|
- tube\.odat\.xyz|
tube\.oisux\.org|
+ tube\.okcinfo\.news|
+ tube\.onlinekirche\.net|
tube\.opportunis\.me|
+ tube\.oraclefilms\.com|
tube\.org\.il|
- tube\.ortion\.xyz|
- tube\.others\.social|
+ tube\.pacapime\.ovh|
+ tube\.parinux\.org|
+ tube\.pastwind\.top|
tube\.picasoft\.net|
- tube\.plomlompom\.com|
+ tube\.pilgerweg-21\.de|
tube\.pmj\.rocks|
+ tube\.pol\.social|
+ tube\.ponsonaille\.fr|
tube\.portes-imaginaire\.org|
+ tube\.public\.apolut\.net|
+ tube\.pustule\.org|
tube\.pyngu\.com|
+ tube\.querdenken-711\.de|
tube\.rebellion\.global|
+ tube\.reseau-canope\.fr|
tube\.rhythms-of-resistance\.org|
- tube\.rita\.moe|
+ tube\.risedsky\.ovh|
+ tube\.rooty\.fr|
tube\.rsi\.cnr\.it|
- tube\.s1gm4\.eu|
- tube\.saumon\.io|
+ tube\.ryne\.moe|
tube\.schleuss\.online|
tube\.schule\.social|
- tube\.seditio\.fr|
+ tube\.sekretaerbaer\.net|
tube\.shanti\.cafe|
tube\.shela\.nu|
tube\.skrep\.in|
+ tube\.sleeping\.town|
tube\.sp-codes\.de|
- tube\.sp4ke\.com|
- tube\.superseriousbusiness\.org|
+ tube\.spdns\.org|
+ tube\.systerserver\.net|
tube\.systest\.eu|
tube\.tappret\.fr|
- tube\.tardis\.world|
- tube\.toontoet\.nl|
+ tube\.techeasy\.org|
+ tube\.thierrytalbert\.fr|
+ tube\.tinfoil-hat\.net|
+ tube\.toldi\.eu|
tube\.tpshd\.de|
+ tube\.trax\.im|
tube\.troopers\.agency|
+ tube\.ttk\.is|
+ tube\.tuxfriend\.fr|
tube\.tylerdavis\.xyz|
+ tube\.ullihome\.de|
+ tube\.ulne\.be|
tube\.undernet\.uy|
- tube\.vigilian-consulting\.nl|
- tube\.vraphim\.com|
- tube\.wehost\.lgbt|
- tube\.wien\.rocks|
+ tube\.vrpnet\.org|
tube\.wolfe\.casa|
tube\.xd0\.de|
+ tube\.xn--baw-joa\.social|
tube\.xy-space\.de|
tube\.yapbreak\.fr|
tubedu\.org|
- tubes\.jodh\.us|
- tuktube\.com|
- turkum\.me|
+ tubulus\.openlatin\.org|
+ turtleisland\.video|
tututu\.tube|
- tuvideo\.encanarias\.info|
- tv1\.cocu\.cc|
- tv1\.gomntu\.space|
- tv2\.cocu\.cc|
+ tv\.adast\.dk|
tv\.adn\.life|
+ tv\.arns\.lt|
tv\.atmx\.ca|
- tv\.bitma\.st|
- tv\.generallyrubbish\.net\.au|
+ tv\.based\.quest|
+ tv\.farewellutopia\.com|
+ tv\.filmfreedom\.net|
+ tv\.gravitons\.org|
+ tv\.io\.seg\.br|
tv\.lumbung\.space|
- tv\.mattchristiansenmedia\.com|
- tv\.netwhood\.online|
- tv\.neue\.city|
- tv\.piejacker\.net|
tv\.pirateradio\.social|
+ tv\.pirati\.cz|
+ tv\.santic-zombie\.ru|
tv\.undersco\.re|
+ tv\.zonepl\.net|
tvox\.ru|
twctube\.twc-zone\.eu|
- unfilter\.tube|
+ twobeek\.com|
+ urbanists\.video|
+ v\.9tail\.net|
v\.basspistol\.org|
+ v\.j4\.lc|
v\.kisombrella\.top|
- v\.lastorder\.xyz|
+ v\.koa\.im|
+ v\.kyaru\.xyz|
v\.lor\.sh|
- v\.phreedom\.club|
- v\.sil\.sh|
- v\.szy\.io|
- v\.xxxapex\.com|
- veezee\.tube|
- vid\.dascoyote\.xyz|
- vid\.garwood\.io|
- vid\.ncrypt\.at|
- vid\.pravdastalina\.info|
- vid\.qorg11\.net|
- vid\.rajeshtaylor\.com|
- vid\.samtripoli\.com|
- vid\.werefox\.dev|
+ v\.mkp\.ca|
+ v\.posm\.gay|
+ v\.slaycer\.top|
+ veedeo\.org|
+ vhs\.absturztau\.be|
+ vid\.cthos\.dev|
+ vid\.kinuseka\.us|
+ vid\.mkp\.ca|
+ vid\.nocogabriel\.fr|
+ vid\.norbipeti\.eu|
+ vid\.northbound\.online|
+ vid\.ohboii\.de|
+ vid\.plantplotting\.co\.uk|
+ vid\.pretok\.tv|
+ vid\.prometheus\.systems|
+ vid\.soafen\.love|
+ vid\.twhtv\.club|
vid\.wildeboer\.net|
video-cave-v2\.de|
+ video-liberty\.com|
video\.076\.ne\.jp|
video\.1146\.nohost\.me|
- video\.altertek\.org|
+ video\.9wd\.eu|
+ video\.abraum\.de|
+ video\.ados\.accoord\.fr|
+ video\.amiga-ng\.org|
video\.anartist\.org|
- video\.apps\.thedoodleproject\.net|
- video\.artist\.cx|
video\.asgardius\.company|
- video\.balsillie\.net|
+ video\.audiovisuel-participatif\.org|
video\.bards\.online|
- video\.binarydad\.com|
+ video\.barkoczy\.social|
+ video\.benetou\.fr|
+ video\.beyondwatts\.social|
+ video\.bgeneric\.net|
+ video\.bilecik\.edu\.tr|
video\.blast-info\.fr|
+ video\.bmu\.cloud|
video\.catgirl\.biz|
+ video\.causa-arcana\.com|
+ video\.chasmcity\.net|
+ video\.chbmeyer\.de|
video\.cigliola\.com|
- video\.cm-en-transition\.fr|
+ video\.citizen4\.eu|
+ video\.clumsy\.computer|
+ video\.cnnumerique\.fr|
+ video\.cnr\.it|
video\.cnt\.social|
video\.coales\.co|
- video\.codingfield\.com|
- video\.comptoir\.net|
video\.comune\.trento\.it|
- video\.cpn\.so|
+ video\.coyp\.us|
video\.csc49\.fr|
- video\.cybre\.town|
- video\.demokratischer-sommer\.de|
- video\.discord-insoumis\.fr|
- video\.dolphincastle\.com|
+ video\.davduf\.net|
+ video\.davejansen\.com|
+ video\.dlearning\.nl|
+ video\.dnfi\.no|
video\.dresden\.network|
- video\.ecole-89\.com|
- video\.elgrillolibertario\.org|
+ video\.drgnz\.club|
+ video\.dudenas\.lt|
+ video\.eientei\.org|
+ video\.ellijaymakerspace\.org|
video\.emergeheart\.info|
video\.eradicatinglove\.xyz|
- video\.ethantheenigma\.me|
- video\.exodus-privacy\.eu\.org|
- video\.fbxl\.net|
+ video\.everythingbagel\.me|
+ video\.extremelycorporate\.ca|
+ video\.fabiomanganiello\.com|
+ video\.fedi\.bzh|
video\.fhtagn\.org|
- video\.greenmycity\.eu|
- video\.guerredeclasse\.fr|
+ video\.firehawk-systems\.com|
+ video\.fox-romka\.ru|
+ video\.fuss\.bz\.it|
+ video\.glassbeadcollective\.org|
+ video\.graine-pdl\.org|
video\.gyt\.is|
- video\.hackers\.town|
+ video\.hainry\.fr|
video\.hardlimit\.com|
- video\.hooli\.co|
+ video\.hostux\.net|
video\.igem\.org|
+ video\.infojournal\.fr|
video\.internet-czas-dzialac\.pl|
+ video\.interru\.io|
+ video\.ipng\.ch|
+ video\.ironsysadmin\.com|
video\.islameye\.com|
- video\.kicik\.fr|
+ video\.jacen\.moe|
+ video\.jadin\.me|
+ video\.jeffmcbride\.net|
+ video\.jigmedatse\.com|
video\.kuba-orlik\.name|
- video\.kyushojitsu\.ca|
+ video\.lacalligramme\.fr|
+ video\.lanceurs-alerte\.fr|
+ video\.laotra\.red|
+ video\.lapineige\.fr|
+ video\.laraffinerie\.re|
video\.lavolte\.net|
- video\.lespoesiesdheloise\.fr|
video\.liberta\.vip|
- video\.liege\.bike|
+ video\.libreti\.net|
+ video\.licentia\.net|
video\.linc\.systems|
video\.linux\.it|
video\.linuxtrent\.it|
- video\.lokal\.social|
+ video\.liveitlive\.show|
video\.lono\.space|
- video\.lunasqu\.ee|
+ video\.lrose\.de|
+ video\.lunago\.net|
video\.lundi\.am|
+ video\.lycee-experimental\.org|
+ video\.maechler\.cloud|
video\.marcorennmaus\.de|
video\.mass-trespass\.uk|
+ video\.matomocamp\.org|
+ video\.medienzentrum-harburg\.de|
+ video\.mentality\.rip|
+ video\.metaversum\.wtf|
+ video\.midreality\.com|
+ video\.mttv\.it|
video\.mugoreve\.fr|
- video\.mundodesconocido\.com|
+ video\.mxtthxw\.art|
video\.mycrowd\.ca|
+ video\.niboe\.info|
video\.nogafam\.es|
- video\.odayacres\.farm|
+ video\.nstr\.no|
+ video\.occm\.cc|
+ video\.off-investigation\.fr|
+ video\.olos311\.org|
+ video\.ordinobsolete\.fr|
+ video\.osvoj\.ru|
+ video\.ourcommon\.cloud|
video\.ozgurkon\.org|
- video\.p1ng0ut\.social|
- video\.p3x\.de|
video\.pcf\.fr|
- video\.pony\.gallery|
- video\.potate\.space|
- video\.pourpenser\.pro|
- video\.progressiv\.dev|
+ video\.pcgaldo\.com|
+ video\.phyrone\.de|
+ video\.poul\.org|
+ video\.publicspaces\.net|
+ video\.pullopen\.xyz|
+ video\.r3s\.nrw|
+ video\.rainevixen\.com|
video\.resolutions\.it|
- video\.rw501\.de|
- video\.screamer\.wiki|
- video\.sdm-tools\.net|
+ video\.retroedge\.tech|
+ video\.rhizome\.org|
+ video\.rlp-media\.de|
+ video\.rs-einrich\.de|
+ video\.rubdos\.be|
+ video\.sadmin\.io|
video\.sftblw\.moe|
video\.shitposter\.club|
- video\.skyn3t\.in|
+ video\.simplex-software\.ru|
+ video\.slipfox\.xyz|
+ video\.snug\.moe|
+ video\.software-fuer-engagierte\.de|
video\.soi\.ch|
- video\.stuartbrand\.co\.uk|
+ video\.sonet\.ws|
+ video\.surazal\.net|
+ video\.taskcards\.eu|
+ video\.team-lcbs\.eu|
+ video\.techforgood\.social|
+ video\.telemillevaches\.net|
+ video\.thepolarbear\.co\.uk|
video\.thinkof\.name|
- video\.toot\.pt|
+ video\.tii\.space|
+ video\.tkz\.es|
+ video\.trankil\.info|
video\.triplea\.fr|
+ video\.tum\.social|
video\.turbo\.chat|
+ video\.uriopss-pdl\.fr|
+ video\.ustim\.ru|
+ video\.ut0pia\.org|
video\.vaku\.org\.ua|
+ video\.vegafjord\.me|
video\.veloma\.org|
video\.violoncello\.ch|
- video\.wilkie\.how|
- video\.wsf2021\.info|
- videorelay\.co|
+ video\.voidconspiracy\.band|
+ video\.wakkeren\.nl|
+ video\.windfluechter\.org|
+ video\.ziez\.eu|
videos-passages\.huma-num\.fr|
- videos\.3d-wolf\.com|
+ videos\.aadtp\.be|
videos\.ahp-numerique\.fr|
- videos\.alexandrebadalo\.pt|
+ videos\.alamaisondulibre\.org|
videos\.archigny\.net|
+ videos\.aroaduntraveled\.com|
+ videos\.b4tech\.org|
videos\.benjaminbrady\.ie|
- videos\.buceoluegoexisto\.com|
- videos\.capas\.se|
- videos\.casually\.cat|
+ videos\.bik\.opencloud\.lu|
videos\.cloudron\.io|
+ videos\.codingotaku\.com|
videos\.coletivos\.org|
+ videos\.collate\.social|
videos\.danksquad\.org|
- videos\.denshi\.live|
- videos\.fromouter\.space|
+ videos\.digitaldragons\.eu|
+ videos\.dromeadhere\.fr|
+ videos\.explain-it\.org|
+ videos\.factsonthegroundshow\.com|
+ videos\.foilen\.com|
videos\.fsci\.in|
+ videos\.gamercast\.net|
+ videos\.gianmarco\.gg|
videos\.globenet\.org|
+ videos\.grafo\.zone|
videos\.hauspie\.fr|
videos\.hush\.is|
+ videos\.hyphalfusion\.network|
+ videos\.icum\.to|
+ videos\.im\.allmendenetz\.de|
+ videos\.jacksonchen666\.com|
videos\.john-livingston\.fr|
- videos\.jordanwarne\.xyz|
- videos\.lavoixdessansvoix\.org|
+ videos\.knazarov\.com|
+ videos\.kuoushi\.com|
+ videos\.laliguepaysdelaloire\.org|
+ videos\.lemouvementassociatif-pdl\.org|
videos\.leslionsfloorball\.fr|
- videos\.lucero\.top|
- videos\.martyn\.berlin|
+ videos\.librescrum\.org|
videos\.mastodont\.cat|
- videos\.monstro1\.com|
- videos\.npo\.city|
- videos\.optoutpod\.com|
- videos\.petch\.rocks|
- videos\.pzelawski\.xyz|
+ videos\.metus\.ca|
+ videos\.miolo\.org|
+ videos\.offroad\.town|
+ videos\.openmandriva\.org|
+ videos\.parleur\.net|
+ videos\.pcorp\.us|
+ videos\.pop\.eu\.com|
videos\.rampin\.org|
+ videos\.rauten\.co\.za|
+ videos\.ritimo\.org|
+ videos\.sarcasmstardust\.com|
videos\.scanlines\.xyz|
videos\.shmalls\.pw|
- videos\.sibear\.fr|
videos\.stadtfabrikanten\.org|
- videos\.tankernn\.eu|
+ videos\.supertuxkart\.net|
videos\.testimonia\.org|
- videos\.thisishowidontdisappear\.com|
- videos\.traumaheilung\.net|
+ videos\.thinkerview\.com|
+ videos\.torrenezzi10\.xyz|
videos\.trom\.tf|
- videos\.wakkerewereld\.nu|
- videos\.weblib\.re|
+ videos\.utsukta\.org|
+ videos\.viorsan\.com|
+ videos\.wherelinux\.xyz|
+ videos\.wikilibriste\.fr|
videos\.yesil\.club|
+ videos\.yeswiki\.net|
+ videotube\.duckdns\.org|
+ vids\.capypara\.de|
vids\.roshless\.me|
+ vids\.stary\.pc\.pl|
vids\.tekdmn\.me|
- vidz\.dou\.bet|
- vod\.lumikko\.dev|
- vs\.uniter\.network|
+ vidz\.julien\.ovh|
+ views\.southfox\.me|
+ virtual-girls-are\.definitely-for\.me|
+ viste\.pt|
+ vnchich\.com|
+ vnop\.org|
+ vod\.newellijay\.tv|
+ voluntarytube\.com|
+ vtr\.chikichiki\.tube|
vulgarisation-informatique\.fr|
- watch\.breadtube\.tv|
- watch\.deranalyst\.ch|
+ watch\.easya\.solutions|
+ watch\.goodluckgabe\.life|
watch\.ignorance\.eu|
- watch\.krazy\.party|
+ watch\.jimmydore\.com|
watch\.libertaria\.space|
- watch\.rt4mn\.org|
- watch\.softinio\.com|
+ watch\.nuked\.social|
+ watch\.ocaml\.org|
+ watch\.thelema\.social|
watch\.tubelab\.video|
web-fellow\.de|
webtv\.vandoeuvre\.net|
- wechill\.space|
+ wetubevid\.online|
wikileaks\.video|
wiwi\.video|
- worldofvids\.com|
- wwtube\.net|
- www4\.mir\.inter21\.net|
- www\.birkeundnymphe\.de|
- www\.captain-german\.com|
- www\.wiki-tube\.de|
+ wow\.such\.disappointment\.fail|
+ www\.jvideos\.net|
+ www\.kotikoff\.net|
+ www\.makertube\.net|
+ www\.mypeer\.tube|
+ www\.nadajemy\.com|
+ www\.neptube\.io|
+ www\.rocaguinarda\.tv|
+ www\.vnshow\.net|
xxivproduction\.video|
- xxx\.noho\.st|
+ yt\.orokoro\.ru|
+ ytube\.retronerd\.at|
+ zumvideo\.de|
# from youtube-dl
peertube\.rainbowswingers\.net|
@@ -1305,24 +1571,6 @@ class PeerTubePlaylistIE(InfoExtractor):
(?P[^/]+)
''' % (PeerTubeIE._INSTANCES_RE, '|'.join(_TYPES.keys()))
_TESTS = [{
- 'url': 'https://peertube.tux.ovh/w/p/3af94cba-95e8-4b74-b37a-807ab6d82526',
- 'info_dict': {
- 'id': '3af94cba-95e8-4b74-b37a-807ab6d82526',
- 'description': 'playlist',
- 'timestamp': 1611171863,
- 'title': 'playlist',
- },
- 'playlist_mincount': 6,
- }, {
- 'url': 'https://peertube.tux.ovh/w/p/wkyqcQBnsvFxtUB2pkYc1e',
- 'info_dict': {
- 'id': 'wkyqcQBnsvFxtUB2pkYc1e',
- 'description': 'Cette liste de vidéos contient uniquement les jeux qui peuvent être terminés en une seule vidéo.',
- 'title': 'Let\'s Play',
- 'timestamp': 1604147331,
- },
- 'playlist_mincount': 6,
- }, {
'url': 'https://peertube.debian.social/w/p/hFdJoTuyhNJVa1cDWd1d12',
'info_dict': {
'id': 'hFdJoTuyhNJVa1cDWd1d12',
From 05420227aaab60a39c0f9ade069c5862be36b1fa Mon Sep 17 00:00:00 2001
From: SirElderling <148036781+SirElderling@users.noreply.github.com>
Date: Mon, 5 Feb 2024 20:39:07 +0000
Subject: [PATCH 126/248] [ie/nytimes] Extract timestamp (#9142)
Authored by: SirElderling
---
yt_dlp/extractor/nytimes.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/yt_dlp/extractor/nytimes.py b/yt_dlp/extractor/nytimes.py
index 354eb02c3..3019202a2 100644
--- a/yt_dlp/extractor/nytimes.py
+++ b/yt_dlp/extractor/nytimes.py
@@ -32,6 +32,7 @@ class NYTimesBaseIE(InfoExtractor):
renderedRepresentation
}
duration
+ firstPublished
promotionalHeadline
promotionalMedia {
... on Image {
@@ -124,6 +125,7 @@ class NYTimesBaseIE(InfoExtractor):
'id': media_id,
'title': data.get('promotionalHeadline'),
'description': data.get('summary'),
+ 'timestamp': parse_iso8601(data.get('firstPublished')),
'duration': float_or_none(data.get('duration'), scale=1000),
'creator': ', '.join(traverse_obj(data, ( # TODO: change to 'creators'
'bylines', ..., 'renderedRepresentation', {lambda x: remove_start(x, 'By ')}))),
@@ -145,8 +147,8 @@ class NYTimesIE(NYTimesBaseIE):
'ext': 'mp4',
'title': 'Verbatim: What Is a Photocopier?',
'description': 'md5:93603dada88ddbda9395632fdc5da260',
- 'timestamp': 1398631707, # FIXME
- 'upload_date': '20140427', # FIXME
+ 'timestamp': 1398646132,
+ 'upload_date': '20140428',
'creator': 'Brett Weiner',
'thumbnail': r're:https?://\w+\.nyt.com/images/.+\.jpg',
'duration': 419,
@@ -310,6 +312,8 @@ class NYTimesCookingIE(NYTimesBaseIE):
'ext': 'mp4',
'title': 'How to Make Mac and Cheese',
'description': 'md5:b8f2f33ec1fb7523b21367147c9594f1',
+ 'timestamp': 1522950315,
+ 'upload_date': '20180405',
'duration': 9.51,
'creator': 'Alison Roman',
'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
From 540b68298192874c75ad5ee4589bed64d02a7d55 Mon Sep 17 00:00:00 2001
From: Dmitry Meyer
Date: Fri, 9 Feb 2024 18:34:56 +0300
Subject: [PATCH 127/248] [ie/Boosty] Add extractor (#9144)
Closes #5900, Closes #8704
Authored by: un-def
---
yt_dlp/extractor/_extractors.py | 1 +
yt_dlp/extractor/boosty.py | 209 ++++++++++++++++++++++++++++++++
2 files changed, 210 insertions(+)
create mode 100644 yt_dlp/extractor/boosty.py
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index e7dd34c77..5d1dd6038 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -257,6 +257,7 @@ from .blogger import BloggerIE
from .bloomberg import BloombergIE
from .bokecc import BokeCCIE
from .bongacams import BongaCamsIE
+from .boosty import BoostyIE
from .bostonglobe import BostonGlobeIE
from .box import BoxIE
from .boxcast import BoxCastVideoIE
diff --git a/yt_dlp/extractor/boosty.py b/yt_dlp/extractor/boosty.py
new file mode 100644
index 000000000..fb14ca146
--- /dev/null
+++ b/yt_dlp/extractor/boosty.py
@@ -0,0 +1,209 @@
+from .common import InfoExtractor
+from .youtube import YoutubeIE
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ qualities,
+ str_or_none,
+ url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class BoostyIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?boosty\.to/(?P[^/#?]+)/posts/(?P[^/#?]+)'
+ _TESTS = [{
+ # single ok_video
+ 'url': 'https://boosty.to/kuplinov/posts/e55d050c-e3bb-4873-a7db-ac7a49b40c38',
+ 'info_dict': {
+ 'id': 'd7473824-352e-48e2-ae53-d4aa39459968',
+ 'title': 'phasma_3',
+ 'channel': 'Kuplinov',
+ 'channel_id': '7958701',
+ 'timestamp': 1655031975,
+ 'upload_date': '20220612',
+ 'release_timestamp': 1655049000,
+ 'release_date': '20220612',
+ 'modified_timestamp': 1668680993,
+ 'modified_date': '20221117',
+ 'tags': ['куплинов', 'phasmophobia'],
+ 'like_count': int,
+ 'ext': 'mp4',
+ 'duration': 105,
+ 'view_count': int,
+ 'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+ },
+ }, {
+ # multiple ok_video
+ 'url': 'https://boosty.to/maddyson/posts/0c652798-3b35-471f-8b48-a76a0b28736f',
+ 'info_dict': {
+ 'id': '0c652798-3b35-471f-8b48-a76a0b28736f',
+ 'title': 'то что не пропустил юта6',
+ 'channel': 'Илья Давыдов',
+ 'channel_id': '6808257',
+ 'timestamp': 1694017040,
+ 'upload_date': '20230906',
+ 'release_timestamp': 1694017040,
+ 'release_date': '20230906',
+ 'modified_timestamp': 1694071178,
+ 'modified_date': '20230907',
+ 'like_count': int,
+ },
+ 'playlist_count': 3,
+ 'playlist': [{
+ 'info_dict': {
+ 'id': 'cc325a9f-a563-41c6-bf47-516c1b506c9a',
+ 'title': 'то что не пропустил юта6',
+ 'channel': 'Илья Давыдов',
+ 'channel_id': '6808257',
+ 'timestamp': 1694017040,
+ 'upload_date': '20230906',
+ 'release_timestamp': 1694017040,
+ 'release_date': '20230906',
+ 'modified_timestamp': 1694071178,
+ 'modified_date': '20230907',
+ 'like_count': int,
+ 'ext': 'mp4',
+ 'duration': 31204,
+ 'view_count': int,
+ 'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+ },
+ }, {
+ 'info_dict': {
+ 'id': 'd07b0a72-9493-4512-b54e-55ce468fd4b7',
+ 'title': 'то что не пропустил юта6',
+ 'channel': 'Илья Давыдов',
+ 'channel_id': '6808257',
+ 'timestamp': 1694017040,
+ 'upload_date': '20230906',
+ 'release_timestamp': 1694017040,
+ 'release_date': '20230906',
+ 'modified_timestamp': 1694071178,
+ 'modified_date': '20230907',
+ 'like_count': int,
+ 'ext': 'mp4',
+ 'duration': 25704,
+ 'view_count': int,
+ 'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+ },
+ }, {
+ 'info_dict': {
+ 'id': '4a3bba32-78c8-422a-9432-2791aff60b42',
+ 'title': 'то что не пропустил юта6',
+ 'channel': 'Илья Давыдов',
+ 'channel_id': '6808257',
+ 'timestamp': 1694017040,
+ 'upload_date': '20230906',
+ 'release_timestamp': 1694017040,
+ 'release_date': '20230906',
+ 'modified_timestamp': 1694071178,
+ 'modified_date': '20230907',
+ 'like_count': int,
+ 'ext': 'mp4',
+ 'duration': 31867,
+ 'view_count': int,
+ 'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+ },
+ }],
+ }, {
+ # single external video (youtube)
+ 'url': 'https://boosty.to/denischuzhoy/posts/6094a487-bcec-4cf8-a453-43313b463c38',
+ 'info_dict': {
+ 'id': 'EXelTnve5lY',
+ 'title': 'Послание Президента Федеральному Собранию | Класс народа',
+ 'upload_date': '20210425',
+ 'channel': 'Денис Чужой',
+ 'tags': 'count:10',
+ 'like_count': int,
+ 'ext': 'mp4',
+ 'duration': 816,
+ 'view_count': int,
+ 'thumbnail': r're:^https://i\.ytimg\.com/',
+ 'age_limit': 0,
+ 'availability': 'public',
+ 'categories': list,
+ 'channel_follower_count': int,
+ 'channel_id': 'UCCzVNbWZfYpBfyofCCUD_0w',
+ 'channel_is_verified': bool,
+ 'channel_url': r're:^https://www\.youtube\.com/',
+ 'comment_count': int,
+ 'description': str,
+ 'heatmap': 'count:100',
+ 'live_status': str,
+ 'playable_in_embed': bool,
+ 'uploader': str,
+ 'uploader_id': str,
+ 'uploader_url': r're:^https://www\.youtube\.com/',
+ },
+ }]
+
+ _MP4_TYPES = ('tiny', 'lowest', 'low', 'medium', 'high', 'full_hd', 'quad_hd', 'ultra_hd')
+
+ def _extract_formats(self, player_urls, video_id):
+ formats = []
+ quality = qualities(self._MP4_TYPES)
+ for player_url in traverse_obj(player_urls, lambda _, v: url_or_none(v['url'])):
+ url = player_url['url']
+ format_type = player_url.get('type')
+ if format_type in ('hls', 'hls_live', 'live_ondemand_hls', 'live_playback_hls'):
+ formats.extend(self._extract_m3u8_formats(url, video_id, m3u8_id='hls', fatal=False))
+ elif format_type in ('dash', 'dash_live', 'live_playback_dash'):
+ formats.extend(self._extract_mpd_formats(url, video_id, mpd_id='dash', fatal=False))
+ elif format_type in self._MP4_TYPES:
+ formats.append({
+ 'url': url,
+ 'ext': 'mp4',
+ 'format_id': format_type,
+ 'quality': quality(format_type),
+ })
+ else:
+ self.report_warning(f'Unknown format type: {format_type!r}')
+ return formats
+
+ def _real_extract(self, url):
+ user, post_id = self._match_valid_url(url).group('user', 'post_id')
+ post = self._download_json(
+ f'https://api.boosty.to/v1/blog/{user}/post/{post_id}', post_id,
+ note='Downloading post data', errnote='Unable to download post data')
+
+ post_title = post.get('title')
+ if not post_title:
+ self.report_warning('Unable to extract post title. Falling back to parsing html page')
+ webpage = self._download_webpage(url, video_id=post_id)
+ post_title = self._og_search_title(webpage, default=None) or self._html_extract_title(webpage)
+
+ common_metadata = {
+ 'title': post_title,
+ **traverse_obj(post, {
+ 'channel': ('user', 'name', {str}),
+ 'channel_id': ('user', 'id', {str_or_none}),
+ 'timestamp': ('createdAt', {int_or_none}),
+ 'release_timestamp': ('publishTime', {int_or_none}),
+ 'modified_timestamp': ('updatedAt', {int_or_none}),
+ 'tags': ('tags', ..., 'title', {str}),
+ 'like_count': ('count', 'likes', {int_or_none}),
+ }),
+ }
+ entries = []
+ for item in traverse_obj(post, ('data', ..., {dict})):
+ item_type = item.get('type')
+ if item_type == 'video' and url_or_none(item.get('url')):
+ entries.append(self.url_result(item['url'], YoutubeIE))
+ elif item_type == 'ok_video':
+ video_id = item.get('id') or post_id
+ entries.append({
+ 'id': video_id,
+ 'formats': self._extract_formats(item.get('playerUrls'), video_id),
+ **common_metadata,
+ **traverse_obj(item, {
+ 'title': ('title', {str}),
+ 'duration': ('duration', {int_or_none}),
+ 'view_count': ('viewsCounter', {int_or_none}),
+ 'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
+ }, get_all=False)})
+
+ if not entries:
+ raise ExtractorError('No videos found', expected=True)
+ if len(entries) == 1:
+ return entries[0]
+ return self.playlist_result(entries, post_id, post_title, **common_metadata)
From 882e3b753c79c7799ce135c3a5edb72494b576af Mon Sep 17 00:00:00 2001
From: "lauren n. liberda"
Date: Sat, 10 Feb 2024 00:11:34 +0100
Subject: [PATCH 128/248] [ie/tvp] Support livestreams (#8860)
Closes #8824
Authored by: selfisekai
---
yt_dlp/extractor/tvp.py | 21 +++++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/yt_dlp/extractor/tvp.py b/yt_dlp/extractor/tvp.py
index 2aa0dd870..a8d00e243 100644
--- a/yt_dlp/extractor/tvp.py
+++ b/yt_dlp/extractor/tvp.py
@@ -21,7 +21,7 @@ from ..utils import (
class TVPIE(InfoExtractor):
IE_NAME = 'tvp'
IE_DESC = 'Telewizja Polska'
- _VALID_URL = r'https?://(?:[^/]+\.)?(?:tvp(?:parlament)?\.(?:pl|info)|tvpworld\.com|swipeto\.pl)/(?:(?!\d+/)[^/]+/)*(?P\d+)'
+ _VALID_URL = r'https?://(?:[^/]+\.)?(?:tvp(?:parlament)?\.(?:pl|info)|tvpworld\.com|swipeto\.pl)/(?:(?!\d+/)[^/]+/)*(?P\d+)(?:[/?#]|$)'
_TESTS = [{
# TVPlayer 2 in js wrapper
@@ -514,7 +514,7 @@ class TVPVODBaseIE(InfoExtractor):
class TVPVODVideoIE(TVPVODBaseIE):
IE_NAME = 'tvp:vod'
- _VALID_URL = r'https?://vod\.tvp\.pl/[a-z\d-]+,\d+/[a-z\d-]+(?\d+)(?:\?[^#]+)?(?:#.+)?$'
+ _VALID_URL = r'https?://vod\.tvp\.pl/(?P[a-z\d-]+,\d+)/[a-z\d-]+(?\d+)/?(?:[?#]|$)'
_TESTS = [{
'url': 'https://vod.tvp.pl/dla-dzieci,24/laboratorium-alchemika-odcinki,309338/odcinek-24,S01E24,311357',
@@ -560,12 +560,23 @@ class TVPVODVideoIE(TVPVODBaseIE):
'thumbnail': 're:https?://.+',
},
'params': {'skip_download': 'm3u8'},
+ }, {
+ 'url': 'https://vod.tvp.pl/live,1/tvp-world,399731',
+ 'info_dict': {
+ 'id': '399731',
+ 'ext': 'mp4',
+ 'title': r're:TVP WORLD \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+ 'live_status': 'is_live',
+ 'thumbnail': 're:https?://.+',
+ },
}]
def _real_extract(self, url):
- video_id = self._match_id(url)
+ category, video_id = self._match_valid_url(url).group('category', 'id')
- info_dict = self._parse_video(self._call_api(f'vods/{video_id}', video_id), with_url=False)
+ is_live = category == 'live,1'
+ entity = 'lives' if is_live else 'vods'
+ info_dict = self._parse_video(self._call_api(f'{entity}/{video_id}', video_id), with_url=False)
playlist = self._call_api(f'{video_id}/videos/playlist', video_id, query={'videoType': 'MOVIE'})
@@ -582,6 +593,8 @@ class TVPVODVideoIE(TVPVODBaseIE):
'ext': 'ttml',
})
+ info_dict['is_live'] = is_live
+
return info_dict
From a1b778428991b1779203bac243ef4e9b6baea90c Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 14:58:18 +0100
Subject: [PATCH 129/248] [build] Move bundle scripts into `bundle` submodule
Authored by: bashonly
---
.github/workflows/build.yml | 20 ++++-----
.github/workflows/release-master.yml | 2 +-
.github/workflows/release-nightly.yml | 2 +-
README.md | 24 ++++++-----
bundle/__init__.py | 1 +
bundle/py2exe.py | 59 +++++++++++++++++++++++++++
pyinst.py => bundle/pyinstaller.py | 2 +-
pyproject.toml | 3 ++
setup.py | 56 +------------------------
9 files changed, 91 insertions(+), 78 deletions(-)
create mode 100644 bundle/__init__.py
create mode 100755 bundle/py2exe.py
rename pyinst.py => bundle/pyinstaller.py (98%)
mode change 100644 => 100755
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 036ce4348..4b05e7cf9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -144,9 +144,9 @@ jobs:
run: |
unset LD_LIBRARY_PATH # Harmful; set by setup-python
conda activate build
- python pyinst.py --onedir
+ python -m bundle.pyinstaller --onedir
(cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
- python pyinst.py
+ python -m bundle.pyinstaller
mv ./dist/yt-dlp_linux ./yt-dlp_linux
mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
@@ -211,7 +211,7 @@ jobs:
python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt # Cached version may be out of date
python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
python3.8 devscripts/make_lazy_extractors.py
- python3.8 pyinst.py
+ python3.8 -m bundle.pyinstaller
if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
@@ -250,9 +250,9 @@ jobs:
python3 devscripts/make_lazy_extractors.py
- name: Build
run: |
- python3 pyinst.py --target-architecture universal2 --onedir
+ python3 -m bundle.pyinstaller --target-architecture universal2 --onedir
(cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
- python3 pyinst.py --target-architecture universal2
+ python3 -m bundle.pyinstaller --target-architecture universal2
- name: Verify --update-to
if: vars.UPDATE_TO_VERIFICATION
@@ -302,7 +302,7 @@ jobs:
python3 devscripts/make_lazy_extractors.py
- name: Build
run: |
- python3 pyinst.py
+ python3 -m bundle.pyinstaller
mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
- name: Verify --update-to
@@ -342,10 +342,10 @@ jobs:
python devscripts/make_lazy_extractors.py
- name: Build
run: |
- python setup.py py2exe
+ python -m bundle.py2exe
Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
- python pyinst.py
- python pyinst.py --onedir
+ python -m bundle.pyinstaller
+ python -m bundle.pyinstaller --onedir
Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
- name: Verify --update-to
@@ -391,7 +391,7 @@ jobs:
python devscripts/make_lazy_extractors.py
- name: Build
run: |
- python pyinst.py
+ python -m bundle.pyinstaller
- name: Verify --update-to
if: vars.UPDATE_TO_VERIFICATION
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 0664137a9..af14b053e 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -7,7 +7,7 @@ on:
- "yt_dlp/**.py"
- "!yt_dlp/version.py"
- "setup.py"
- - "pyinst.py"
+ - "bundle/*.py"
concurrency:
group: release-master
permissions:
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 2e623a67c..3f1418936 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -18,7 +18,7 @@ jobs:
- name: Check for new commits
id: check_for_new_commits
run: |
- relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "pyinst.py")
+ relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "bundle/*.py")
echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
release:
diff --git a/README.md b/README.md
index 7dc3bb2f6..c74777d2f 100644
--- a/README.md
+++ b/README.md
@@ -321,19 +321,21 @@ If you do not have the necessary dependencies for a task you are attempting, yt-
## COMPILE
### Standalone PyInstaller Builds
-To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). Once you have all the necessary dependencies installed, simply run `pyinst.py`. The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used.
+To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used. You can run the following commands:
- python3 -m pip install -U pyinstaller -r requirements.txt
- python3 devscripts/make_lazy_extractors.py
- python3 pyinst.py
+```
+python3 -m pip install -U pyinstaller -r requirements.txt
+python3 devscripts/make_lazy_extractors.py
+python3 -m bundle.pyinstaller
+```
On some systems, you may need to use `py` or `python` instead of `python3`.
-`pyinst.py` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate).
+`bundle/pyinstaller.py` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate).
**Note**: Pyinstaller versions below 4.4 [do not support](https://github.com/pyinstaller/pyinstaller#requirements-and-tested-platforms) Python installed from the Windows store without using a virtual environment.
-**Important**: Running `pyinstaller` directly **without** using `pyinst.py` is **not** officially supported. This may or may not work correctly.
+**Important**: Running `pyinstaller` directly **without** using `bundle/pyinstaller.py` is **not** officially supported. This may or may not work correctly.
### Platform-independent Binary (UNIX)
You will need the build tools `python` (3.8+), `zip`, `make` (GNU), `pandoc`\* and `pytest`\*.
@@ -346,11 +348,13 @@ You can also run `make yt-dlp` instead to compile only the binary without updati
While we provide the option to build with [py2exe](https://www.py2exe.org), it is recommended to build [using PyInstaller](#standalone-pyinstaller-builds) instead since the py2exe builds **cannot contain `pycryptodomex`/`certifi` and needs VC++14** on the target computer to run.
-If you wish to build it anyway, install Python and py2exe, and then simply run `setup.py py2exe`
+If you wish to build it anyway, install Python (if it is not already installed) and you can run the following commands:
- py -m pip install -U py2exe -r requirements.txt
- py devscripts/make_lazy_extractors.py
- py setup.py py2exe
+```
+py -m pip install -U py2exe -r requirements.txt
+py devscripts/make_lazy_extractors.py
+py -m bundle.py2exe
+```
### Related scripts
diff --git a/bundle/__init__.py b/bundle/__init__.py
new file mode 100644
index 000000000..932b79829
--- /dev/null
+++ b/bundle/__init__.py
@@ -0,0 +1 @@
+# Empty file
diff --git a/bundle/py2exe.py b/bundle/py2exe.py
new file mode 100755
index 000000000..a7e4113f1
--- /dev/null
+++ b/bundle/py2exe.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+# Allow execution from anywhere
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import warnings
+
+from py2exe import freeze
+
+from devscripts.utils import read_version
+
+VERSION = read_version()
+
+
+def main():
+ warnings.warn(
+ 'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
+ 'It is recommended to run "pyinst.py" to build using pyinstaller instead')
+
+ return freeze(
+ console=[{
+ 'script': './yt_dlp/__main__.py',
+ 'dest_base': 'yt-dlp',
+ 'icon_resources': [(1, 'devscripts/logo.ico')],
+ }],
+ version_info={
+ 'version': VERSION,
+ 'description': 'A youtube-dl fork with additional features and patches',
+ 'comments': 'Official repository: ',
+ 'product_name': 'yt-dlp',
+ 'product_version': VERSION,
+ },
+ options={
+ 'bundle_files': 0,
+ 'compressed': 1,
+ 'optimize': 2,
+ 'dist_dir': './dist',
+ 'excludes': [
+ # py2exe cannot import Crypto
+ 'Crypto',
+ 'Cryptodome',
+ # py2exe appears to confuse this with our socks library.
+ # We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
+ 'urllib3.contrib.socks'
+ ],
+ 'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
+ # Modules that are only imported dynamically must be added here
+ 'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
+ 'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
+ },
+ zipfile=None,
+ )
+
+
+if __name__ == '__main__':
+ main()
diff --git a/pyinst.py b/bundle/pyinstaller.py
old mode 100644
new mode 100755
similarity index 98%
rename from pyinst.py
rename to bundle/pyinstaller.py
index c36f6acd4..db9dbfde5
--- a/pyinst.py
+++ b/bundle/pyinstaller.py
@@ -4,7 +4,7 @@
import os
import sys
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import platform
diff --git a/pyproject.toml b/pyproject.toml
index 97718ec43..626d9aa13 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,3 +3,6 @@ build-backend = 'setuptools.build_meta'
# https://github.com/yt-dlp/yt-dlp/issues/5941
# https://github.com/pypa/distutils/issues/17
requires = ['setuptools > 50']
+
+[project.entry-points.pyinstaller40]
+hook-dirs = "yt_dlp.__pyinstaller:get_hook_dirs"
diff --git a/setup.py b/setup.py
index 3d9a69d10..fc5b50468 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,6 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import subprocess
-import warnings
try:
from setuptools import Command, find_packages, setup
@@ -39,46 +38,6 @@ def packages():
]
-def py2exe_params():
- warnings.warn(
- 'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
- 'It is recommended to run "pyinst.py" to build using pyinstaller instead')
-
- return {
- 'console': [{
- 'script': './yt_dlp/__main__.py',
- 'dest_base': 'yt-dlp',
- 'icon_resources': [(1, 'devscripts/logo.ico')],
- }],
- 'version_info': {
- 'version': VERSION,
- 'description': DESCRIPTION,
- 'comments': LONG_DESCRIPTION.split('\n')[0],
- 'product_name': 'yt-dlp',
- 'product_version': VERSION,
- },
- 'options': {
- 'bundle_files': 0,
- 'compressed': 1,
- 'optimize': 2,
- 'dist_dir': './dist',
- 'excludes': [
- # py2exe cannot import Crypto
- 'Crypto',
- 'Cryptodome',
- # py2exe appears to confuse this with our socks library.
- # We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
- 'urllib3.contrib.socks'
- ],
- 'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
- # Modules that are only imported dynamically must be added here
- 'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
- 'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
- },
- 'zipfile': None,
- }
-
-
def build_params():
files_spec = [
('share/bash-completion/completions', ['completions/bash/yt-dlp']),
@@ -127,20 +86,7 @@ class build_lazy_extractors(Command):
def main():
- if sys.argv[1:2] == ['py2exe']:
- params = py2exe_params()
- try:
- from py2exe import freeze
- except ImportError:
- import py2exe # noqa: F401
- warnings.warn('You are using an outdated version of py2exe. Support for this version will be removed in the future')
- params['console'][0].update(params.pop('version_info'))
- params['options'] = {'py2exe': params.pop('options')}
- else:
- return freeze(**params)
- else:
- params = build_params()
-
+ params = build_params()
setup(
name='yt-dlp', # package name (do not change/remove comment)
version=VERSION,
From 868d2f60a7cb59b410c8cbfb452cbdb072687b81 Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 15:07:45 +0100
Subject: [PATCH 130/248] [build:Makefile] Add automated `CODE_FOLDERS` and
`CODE_FILES`
Authored by: bashonly
---
Makefile | 27 ++++++++++++---------------
1 file changed, 12 insertions(+), 15 deletions(-)
diff --git a/Makefile b/Makefile
index c85b24c13..296fc3260 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,7 @@ clean-test:
*.mp4 *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
clean-dist:
rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
- yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
+ yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS
clean-cache:
find . \( \
-type d -name .pytest_cache -o -type d -name __pycache__ -o -name "*.pyc" -o -name "*.class" \
@@ -73,24 +73,24 @@ test:
offlinetest: codetest
$(PYTHON) -m pytest -k "not download"
-# XXX: This is hard to maintain
-CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies yt_dlp/networking
-yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
+CODE_FOLDERS := $(shell find yt_dlp -type d -not -name '__*' -exec sh -c 'test -e "$$1"/__init__.py' sh {} \; -print)
+CODE_FILES := $(shell for f in $(CODE_FOLDERS); do echo "$$f" | awk '{gsub(/\/[^\/]+/,"/*"); print $$1"/*.py"}'; done | sort -u)
+yt-dlp: $(CODE_FILES)
mkdir -p zip
for d in $(CODE_FOLDERS) ; do \
mkdir -p zip/$$d ;\
cp -pPR $$d/*.py zip/$$d/ ;\
done
- touch -t 200001010101 zip/yt_dlp/*.py zip/yt_dlp/*/*.py
+ cd zip ; touch -t 200001010101 $(CODE_FILES)
mv zip/yt_dlp/__main__.py zip/
- cd zip ; zip -q ../yt-dlp yt_dlp/*.py yt_dlp/*/*.py __main__.py
+ cd zip ; zip -q ../yt-dlp $(CODE_FILES) __main__.py
rm -rf zip
echo '#!$(PYTHON)' > yt-dlp
cat yt-dlp.zip >> yt-dlp
rm yt-dlp.zip
chmod a+x yt-dlp
-README.md: yt_dlp/*.py yt_dlp/*/*.py devscripts/make_readme.py
+README.md: $(CODE_FILES) devscripts/make_readme.py
COLUMNS=80 $(PYTHON) yt_dlp/__main__.py --ignore-config --help | $(PYTHON) devscripts/make_readme.py
CONTRIBUTING.md: README.md devscripts/make_contributing.py
@@ -115,15 +115,15 @@ yt-dlp.1: README.md devscripts/prepare_manpage.py
pandoc -s -f $(MARKDOWN) -t man yt-dlp.1.temp.md -o yt-dlp.1
rm -f yt-dlp.1.temp.md
-completions/bash/yt-dlp: yt_dlp/*.py yt_dlp/*/*.py devscripts/bash-completion.in
+completions/bash/yt-dlp: $(CODE_FILES) devscripts/bash-completion.in
mkdir -p completions/bash
$(PYTHON) devscripts/bash-completion.py
-completions/zsh/_yt-dlp: yt_dlp/*.py yt_dlp/*/*.py devscripts/zsh-completion.in
+completions/zsh/_yt-dlp: $(CODE_FILES) devscripts/zsh-completion.in
mkdir -p completions/zsh
$(PYTHON) devscripts/zsh-completion.py
-completions/fish/yt-dlp.fish: yt_dlp/*.py yt_dlp/*/*.py devscripts/fish-completion.in
+completions/fish/yt-dlp.fish: $(CODE_FILES) devscripts/fish-completion.in
mkdir -p completions/fish
$(PYTHON) devscripts/fish-completion.py
@@ -148,8 +148,5 @@ yt-dlp.tar.gz: all
setup.py setup.cfg yt-dlp yt_dlp requirements.txt \
devscripts test
-AUTHORS: .mailmap
- git shortlog -s -n | cut -f2 | sort > AUTHORS
-
-.mailmap:
- git shortlog -s -e -n | awk '!(out[$$NF]++) { $$1="";sub(/^[ \t]+/,""); print}' > .mailmap
+AUTHORS:
+ git shortlog -s -n HEAD | cut -f2 | sort > AUTHORS
From 775cde82dc5b1dc64ab0539a92dd8c7ba6c0ad33 Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 15:13:03 +0100
Subject: [PATCH 131/248] [build] Migrate to `pyproject.toml` and `hatchling`
Authored by: bashonly
---
.github/workflows/release-master.yml | 2 +-
.github/workflows/release-nightly.yml | 2 +-
.github/workflows/release.yml | 9 +-
MANIFEST.in | 10 --
Makefile | 11 +--
pyproject.toml | 120 +++++++++++++++++++++++-
setup.cfg | 4 -
setup.py | 129 --------------------------
8 files changed, 130 insertions(+), 157 deletions(-)
delete mode 100644 MANIFEST.in
delete mode 100644 setup.py
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index af14b053e..2430dc5f8 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -6,8 +6,8 @@ on:
paths:
- "yt_dlp/**.py"
- "!yt_dlp/version.py"
- - "setup.py"
- "bundle/*.py"
+ - "pyproject.toml"
concurrency:
group: release-master
permissions:
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 3f1418936..16d583846 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -18,7 +18,7 @@ jobs:
- name: Check for new commits
id: check_for_new_commits
run: |
- relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "setup.py" "bundle/*.py")
+ relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "bundle/*.py" "pyproject.toml")
echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
release:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 69b5e3152..d1508e5e6 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -266,14 +266,19 @@ jobs:
run: |
python devscripts/update-version.py -c "${{ env.channel }}" -r "${{ env.target_repo }}" -s "${{ env.suffix }}" "${{ env.version }}"
python devscripts/make_lazy_extractors.py
- sed -i -E "s/(name=')[^']+(', # package name)/\1${{ env.pypi_project }}\2/" setup.py
+ sed -i -E '0,/(name = ")[^"]+(")/s//\1${{ env.pypi_project }}\2/' pyproject.toml
- name: Build
run: |
rm -rf dist/*
make pypi-files
+ printf '%s\n\n' \
+ 'Official repository: ' \
+ '**PS**: Some links in this document will not work since this is a copy of the README.md from Github' > ./README.md.new
+ cat ./README.md >> ./README.md.new && mv -f ./README.md.new ./README.md
python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
- python setup.py sdist bdist_wheel
+ make clean-cache
+ python -m build --no-isolation .
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index bc2f056c0..000000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,10 +0,0 @@
-include AUTHORS
-include Changelog.md
-include LICENSE
-include README.md
-include completions/*/*
-include supportedsites.md
-include yt-dlp.1
-include requirements.txt
-recursive-include devscripts *
-recursive-include test *
diff --git a/Makefile b/Makefile
index 296fc3260..2f36c0cd1 100644
--- a/Makefile
+++ b/Makefile
@@ -6,11 +6,11 @@ doc: README.md CONTRIBUTING.md issuetemplates supportedsites
ot: offlinetest
tar: yt-dlp.tar.gz
-# Keep this list in sync with MANIFEST.in
+# Keep this list in sync with pyproject.toml includes/artifacts
# intended use: when building a source distribution,
-# make pypi-files && python setup.py sdist
+# make pypi-files && python3 -m build -sn .
pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \
- completions yt-dlp.1 requirements.txt setup.cfg devscripts/* test/*
+ completions yt-dlp.1 pyproject.toml setup.cfg devscripts/* test/*
.PHONY: all clean install test tar pypi-files completions ot offlinetest codetest supportedsites
@@ -144,9 +144,8 @@ yt-dlp.tar.gz: all
-- \
README.md supportedsites.md Changelog.md LICENSE \
CONTRIBUTING.md Collaborators.md CONTRIBUTORS AUTHORS \
- Makefile MANIFEST.in yt-dlp.1 README.txt completions \
- setup.py setup.cfg yt-dlp yt_dlp requirements.txt \
- devscripts test
+ Makefile yt-dlp.1 README.txt completions .gitignore \
+ setup.cfg yt-dlp yt_dlp pyproject.toml devscripts test
AUTHORS:
git shortlog -s -n HEAD | cut -f2 | sort > AUTHORS
diff --git a/pyproject.toml b/pyproject.toml
index 626d9aa13..5ef013279 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,8 +1,120 @@
[build-system]
-build-backend = 'setuptools.build_meta'
-# https://github.com/yt-dlp/yt-dlp/issues/5941
-# https://github.com/pypa/distutils/issues/17
-requires = ['setuptools > 50']
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "yt-dlp"
+maintainers = [
+ {name = "pukkandan", email = "pukkandan.ytdlp@gmail.com"},
+ {name = "Grub4K", email = "contact@grub4k.xyz"},
+ {name = "bashonly", email = "bashonly@protonmail.com"},
+]
+description = "A youtube-dl fork with additional features and patches"
+readme = "README.md"
+requires-python = ">=3.8"
+keywords = [
+ "youtube-dl",
+ "video-downloader",
+ "youtube-downloader",
+ "sponsorblock",
+ "youtube-dlc",
+ "yt-dlp",
+]
+license = {file = "LICENSE"}
+classifiers = [
+ "Topic :: Multimedia :: Video",
+ "Development Status :: 5 - Production/Stable",
+ "Environment :: Console",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3 :: Only",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: Implementation",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy",
+ "License :: OSI Approved :: The Unlicense (Unlicense)",
+ "Operating System :: OS Independent",
+]
+dynamic = ["version"]
+dependencies = [
+ "brotli; implementation_name=='cpython'",
+ "brotlicffi; implementation_name!='cpython'",
+ "certifi",
+ "mutagen",
+ "pycryptodomex",
+ "requests>=2.31.0,<3",
+ "urllib3>=1.26.17,<3",
+ "websockets>=12.0",
+]
+
+[project.optional-dependencies]
+secretstorage = [
+ "cffi",
+ "secretstorage",
+]
+build = [
+ "build",
+ "hatchling",
+ "pip",
+ "wheel",
+]
+dev = [
+ "flake8",
+ "isort",
+ "pytest",
+]
+pyinstaller = ["pyinstaller>=6.3"]
+py2exe = ["py2exe>=0.12"]
+
+[project.urls]
+Documentation = "https://github.com/yt-dlp/yt-dlp#readme"
+Repository = "https://github.com/yt-dlp/yt-dlp"
+Tracker = "https://github.com/yt-dlp/yt-dlp/issues"
+Funding = "https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators"
+
+[project.scripts]
+yt-dlp = "yt_dlp:main"
[project.entry-points.pyinstaller40]
hook-dirs = "yt_dlp.__pyinstaller:get_hook_dirs"
+
+[tool.hatch.build.targets.sdist]
+include = [
+ "/yt_dlp",
+ "/devscripts",
+ "/test",
+ "/.gitignore", # included by default, needed for auto-excludes
+ "/Changelog.md",
+ "/LICENSE", # included as license
+ "/pyproject.toml", # included by default
+ "/README.md", # included as readme
+ "/setup.cfg",
+ "/supportedsites.md",
+]
+exclude = ["/yt_dlp/__pyinstaller"]
+artifacts = [
+ "/yt_dlp/extractor/lazy_extractors.py",
+ "/completions",
+ "/AUTHORS", # included by default
+ "/README.txt",
+ "/yt-dlp.1",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["yt_dlp"]
+exclude = ["/yt_dlp/__pyinstaller"]
+artifacts = ["/yt_dlp/extractor/lazy_extractors.py"]
+
+[tool.hatch.build.targets.wheel.shared-data]
+"completions/bash/yt-dlp" = "share/bash-completion/completions/yt-dlp"
+"completions/zsh/_yt-dlp" = "share/zsh/site-functions/_yt-dlp"
+"completions/fish/yt-dlp.fish" = "share/fish/vendor_completions.d/yt-dlp.fish"
+"README.txt" = "share/doc/yt_dlp/README.txt"
+"yt-dlp.1" = "share/man/man1/yt-dlp.1"
+
+[tool.hatch.version]
+path = "yt_dlp/version.py"
+pattern = "_pkg_version = '(?P[^']+)'"
diff --git a/setup.cfg b/setup.cfg
index a799f7293..aeb4cee58 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,7 +1,3 @@
-[wheel]
-universal = true
-
-
[flake8]
exclude = build,venv,.tox,.git,.pytest_cache
ignore = E402,E501,E731,E741,W503
diff --git a/setup.py b/setup.py
deleted file mode 100644
index fc5b50468..000000000
--- a/setup.py
+++ /dev/null
@@ -1,129 +0,0 @@
-#!/usr/bin/env python3
-
-# Allow execution from anywhere
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-import subprocess
-
-try:
- from setuptools import Command, find_packages, setup
- setuptools_available = True
-except ImportError:
- from distutils.core import Command, setup
- setuptools_available = False
-
-from devscripts.utils import read_file, read_version
-
-VERSION = read_version(varname='_pkg_version')
-
-DESCRIPTION = 'A youtube-dl fork with additional features and patches'
-
-LONG_DESCRIPTION = '\n\n'.join((
- 'Official repository: ',
- '**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
- read_file('README.md')))
-
-REQUIREMENTS = read_file('requirements.txt').splitlines()
-
-
-def packages():
- if setuptools_available:
- return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'))
-
- return [
- 'yt_dlp', 'yt_dlp.extractor', 'yt_dlp.downloader', 'yt_dlp.postprocessor', 'yt_dlp.compat',
- ]
-
-
-def build_params():
- files_spec = [
- ('share/bash-completion/completions', ['completions/bash/yt-dlp']),
- ('share/zsh/site-functions', ['completions/zsh/_yt-dlp']),
- ('share/fish/vendor_completions.d', ['completions/fish/yt-dlp.fish']),
- ('share/doc/yt_dlp', ['README.txt']),
- ('share/man/man1', ['yt-dlp.1'])
- ]
- data_files = []
- for dirname, files in files_spec:
- resfiles = []
- for fn in files:
- if not os.path.exists(fn):
- warnings.warn(f'Skipping file {fn} since it is not present. Try running " make pypi-files " first')
- else:
- resfiles.append(fn)
- data_files.append((dirname, resfiles))
-
- params = {'data_files': data_files}
-
- if setuptools_available:
- params['entry_points'] = {
- 'console_scripts': ['yt-dlp = yt_dlp:main'],
- 'pyinstaller40': ['hook-dirs = yt_dlp.__pyinstaller:get_hook_dirs'],
- }
- else:
- params['scripts'] = ['yt-dlp']
- return params
-
-
-class build_lazy_extractors(Command):
- description = 'Build the extractor lazy loading module'
- user_options = []
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- pass
-
- def run(self):
- if self.dry_run:
- print('Skipping build of lazy extractors in dry run mode')
- return
- subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])
-
-
-def main():
- params = build_params()
- setup(
- name='yt-dlp', # package name (do not change/remove comment)
- version=VERSION,
- maintainer='pukkandan',
- maintainer_email='pukkandan.ytdlp@gmail.com',
- description=DESCRIPTION,
- long_description=LONG_DESCRIPTION,
- long_description_content_type='text/markdown',
- url='https://github.com/yt-dlp/yt-dlp',
- packages=packages(),
- install_requires=REQUIREMENTS,
- python_requires='>=3.8',
- project_urls={
- 'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
- 'Source': 'https://github.com/yt-dlp/yt-dlp',
- 'Tracker': 'https://github.com/yt-dlp/yt-dlp/issues',
- 'Funding': 'https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators',
- },
- classifiers=[
- 'Topic :: Multimedia :: Video',
- 'Development Status :: 5 - Production/Stable',
- 'Environment :: Console',
- 'Programming Language :: Python',
- 'Programming Language :: Python :: 3.8',
- 'Programming Language :: Python :: 3.9',
- 'Programming Language :: Python :: 3.10',
- 'Programming Language :: Python :: 3.11',
- 'Programming Language :: Python :: 3.12',
- 'Programming Language :: Python :: Implementation',
- 'Programming Language :: Python :: Implementation :: CPython',
- 'Programming Language :: Python :: Implementation :: PyPy',
- 'License :: Public Domain',
- 'Operating System :: OS Independent',
- ],
- cmdclass={'build_lazy_extractors': build_lazy_extractors},
- **params
- )
-
-
-main()
From fd647775e27e030ab17387c249e2ebeba68f8ff0 Mon Sep 17 00:00:00 2001
From: Simon Sawicki
Date: Sun, 11 Feb 2024 15:14:42 +0100
Subject: [PATCH 132/248] [devscripts] `tomlparse`: Add makeshift toml parser
Authored by: Grub4K
---
devscripts/tomlparse.py | 189 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 189 insertions(+)
create mode 100755 devscripts/tomlparse.py
diff --git a/devscripts/tomlparse.py b/devscripts/tomlparse.py
new file mode 100755
index 000000000..85ac4eef7
--- /dev/null
+++ b/devscripts/tomlparse.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+
+"""
+Simple parser for spec compliant toml files
+
+A simple toml parser for files that comply with the spec.
+Should only be used to parse `pyproject.toml` for `install_deps.py`.
+
+IMPORTANT: INVALID FILES OR MULTILINE STRINGS ARE NOT SUPPORTED!
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+import re
+
+WS = r'(?:[\ \t]*)'
+STRING_RE = re.compile(r'"(?:\\.|[^\\"\n])*"|\'[^\'\n]*\'')
+SINGLE_KEY_RE = re.compile(rf'{STRING_RE.pattern}|[A-Za-z0-9_-]+')
+KEY_RE = re.compile(rf'{WS}(?:{SINGLE_KEY_RE.pattern}){WS}(?:\.{WS}(?:{SINGLE_KEY_RE.pattern}){WS})*')
+EQUALS_RE = re.compile(rf'={WS}')
+WS_RE = re.compile(WS)
+
+_SUBTABLE = rf'(?P^\[(?P\[)?(?P{KEY_RE.pattern})\]\]?)'
+EXPRESSION_RE = re.compile(rf'^(?:{_SUBTABLE}|{KEY_RE.pattern}=)', re.MULTILINE)
+
+LIST_WS_RE = re.compile(rf'{WS}((#[^\n]*)?\n{WS})*')
+LEFTOVER_VALUE_RE = re.compile(r'[^,}\]\t\n#]+')
+
+
+def parse_key(value: str):
+ for match in SINGLE_KEY_RE.finditer(value):
+ if match[0][0] == '"':
+ yield json.loads(match[0])
+ elif match[0][0] == '\'':
+ yield match[0][1:-1]
+ else:
+ yield match[0]
+
+
+def get_target(root: dict, paths: list[str], is_list=False):
+ target = root
+
+ for index, key in enumerate(paths, 1):
+ use_list = is_list and index == len(paths)
+ result = target.get(key)
+ if result is None:
+ result = [] if use_list else {}
+ target[key] = result
+
+ if isinstance(result, dict):
+ target = result
+ elif use_list:
+ target = {}
+ result.append(target)
+ else:
+ target = result[-1]
+
+ assert isinstance(target, dict)
+ return target
+
+
+def parse_enclosed(data: str, index: int, end: str, ws_re: re.Pattern):
+ index += 1
+
+ if match := ws_re.match(data, index):
+ index = match.end()
+
+ while data[index] != end:
+ index = yield True, index
+
+ if match := ws_re.match(data, index):
+ index = match.end()
+
+ if data[index] == ',':
+ index += 1
+
+ if match := ws_re.match(data, index):
+ index = match.end()
+
+ assert data[index] == end
+ yield False, index + 1
+
+
+def parse_value(data: str, index: int):
+ if data[index] == '[':
+ result = []
+
+ indices = parse_enclosed(data, index, ']', LIST_WS_RE)
+ valid, index = next(indices)
+ while valid:
+ index, value = parse_value(data, index)
+ result.append(value)
+ valid, index = indices.send(index)
+
+ return index, result
+
+ if data[index] == '{':
+ result = {}
+
+ indices = parse_enclosed(data, index, '}', WS_RE)
+ valid, index = next(indices)
+ while valid:
+ valid, index = indices.send(parse_kv_pair(data, index, result))
+
+ return index, result
+
+ if match := STRING_RE.match(data, index):
+ return match.end(), json.loads(match[0]) if match[0][0] == '"' else match[0][1:-1]
+
+ match = LEFTOVER_VALUE_RE.match(data, index)
+ assert match
+ value = match[0].strip()
+ for func in [
+ int,
+ float,
+ datetime.time.fromisoformat,
+ datetime.date.fromisoformat,
+ datetime.datetime.fromisoformat,
+ {'true': True, 'false': False}.get,
+ ]:
+ try:
+ value = func(value)
+ break
+ except Exception:
+ pass
+
+ return match.end(), value
+
+
+def parse_kv_pair(data: str, index: int, target: dict):
+ match = KEY_RE.match(data, index)
+ if not match:
+ return None
+
+ *keys, key = parse_key(match[0])
+
+ match = EQUALS_RE.match(data, match.end())
+ assert match
+ index = match.end()
+
+ index, value = parse_value(data, index)
+ get_target(target, keys)[key] = value
+ return index
+
+
+def parse_toml(data: str):
+ root = {}
+ target = root
+
+ index = 0
+ while True:
+ match = EXPRESSION_RE.search(data, index)
+ if not match:
+ break
+
+ if match.group('subtable'):
+ index = match.end()
+ path, is_list = match.group('path', 'is_list')
+ target = get_target(root, list(parse_key(path)), bool(is_list))
+ continue
+
+ index = parse_kv_pair(data, match.start(), target)
+ assert index is not None
+
+ return root
+
+
+def main():
+ import argparse
+ from pathlib import Path
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('infile', type=Path, help='The TOML file to read as input')
+ args = parser.parse_args()
+
+ with args.infile.open('r', encoding='utf-8') as file:
+ data = file.read()
+
+ def default(obj):
+ if isinstance(obj, (datetime.date, datetime.time, datetime.datetime)):
+ return obj.isoformat()
+
+ print(json.dumps(parse_toml(data), default=default))
+
+
+if __name__ == '__main__':
+ main()
From b8a433aaca86b15cb9f1a451b0f69371d2fc22a9 Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 15:17:08 +0100
Subject: [PATCH 133/248] [devscripts] `install_deps`: Add script and migrate
to it
Authored by: bashonly
---
.github/workflows/build.yml | 36 +++++++++--------
.github/workflows/core.yml | 2 +-
.github/workflows/download.yml | 4 +-
.github/workflows/quick-test.yml | 6 +--
.github/workflows/release.yml | 3 +-
README.md | 5 ++-
devscripts/install_deps.py | 66 ++++++++++++++++++++++++++++++++
requirements.txt | 8 ----
8 files changed, 95 insertions(+), 35 deletions(-)
create mode 100755 devscripts/install_deps.py
delete mode 100644 requirements.txt
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4b05e7cf9..082164c9e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -121,16 +121,14 @@ jobs:
- name: Install Requirements
run: |
sudo apt -y install zip pandoc man sed
- reqs=$(mktemp)
- cat > "$reqs" << EOF
+ cat > ./requirements.txt << EOF
python=3.10.*
- pyinstaller
- cffi
brotli-python
- secretstorage
EOF
- sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
- mamba create -n build --file "$reqs"
+ python devscripts/install_deps.py --print \
+ --exclude brotli --exclude brotlicffi \
+ --include secretstorage --include pyinstaller >> ./requirements.txt
+ mamba create -n build --file ./requirements.txt
- name: Prepare
run: |
@@ -203,12 +201,13 @@ jobs:
apt update
apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
python3.8 -m pip install -U pip setuptools wheel
- # Cannot access requirements.txt from the repo directory at this stage
+ # Cannot access any files from the repo directory at this stage
python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
run: |
cd repo
- python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt # Cached version may be out of date
+ python3.8 devscripts/install_deps.py -o --include build
+ python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage # Cached version may be out of date
python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
python3.8 devscripts/make_lazy_extractors.py
python3.8 -m bundle.pyinstaller
@@ -240,9 +239,10 @@ jobs:
- name: Install Requirements
run: |
brew install coreutils
- python3 -m pip install -U --user pip setuptools wheel
+ python3 devscripts/install_deps.py --user -o --include build
+ python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
# We need to ignore wheels otherwise we break universal2 builds
- python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
+ python3 -m pip install -U --user --no-binary :all: -r requirements.txt
- name: Prepare
run: |
@@ -293,8 +293,8 @@ jobs:
- name: Install Requirements
run: |
brew install coreutils
- python3 -m pip install -U --user pip setuptools wheel
- python3 -m pip install -U --user Pyinstaller -r requirements.txt
+ python3 devscripts/install_deps.py --user -o --include build
+ python3 devscripts/install_deps.py --user --include pyinstaller
- name: Prepare
run: |
@@ -333,8 +333,9 @@ jobs:
python-version: "3.8"
- name: Install Requirements
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
- python -m pip install -U pip setuptools wheel py2exe
- pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
+ python devscripts/install_deps.py -o --include build
+ python devscripts/install_deps.py --include py2exe
+ python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
- name: Prepare
run: |
@@ -382,8 +383,9 @@ jobs:
architecture: "x86"
- name: Install Requirements
run: |
- python -m pip install -U pip setuptools wheel
- pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
+ python devscripts/install_deps.py -o --include build
+ python devscripts/install_deps.py
+ python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl"
- name: Prepare
run: |
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index eaaf03dee..f694c9bdd 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -53,7 +53,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Install test requirements
- run: pip install pytest -r requirements.txt
+ run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
continue-on-error: False
run: |
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 9f47d6718..84339d970 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -15,7 +15,7 @@ jobs:
with:
python-version: 3.9
- name: Install test requirements
- run: pip install pytest -r requirements.txt
+ run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
continue-on-error: true
run: python3 ./devscripts/run_tests.py download
@@ -42,7 +42,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Install test requirements
- run: pip install pytest -r requirements.txt
+ run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
continue-on-error: true
run: python3 ./devscripts/run_tests.py download
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index 84fca62d4..4e9616926 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -15,7 +15,7 @@ jobs:
with:
python-version: '3.8'
- name: Install test requirements
- run: pip install pytest -r requirements.txt
+ run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
run: |
python3 -m yt_dlp -v || true
@@ -28,8 +28,8 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
- name: Install flake8
- run: pip install flake8
+ run: python3 ./devscripts/install_deps.py -o --include dev
- name: Make lazy extractors
- run: python devscripts/make_lazy_extractors.py
+ run: python3 ./devscripts/make_lazy_extractors.py
- name: Run flake8
run: flake8 .
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d1508e5e6..1653add4f 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -253,8 +253,7 @@ jobs:
- name: Install Requirements
run: |
sudo apt -y install pandoc man
- python -m pip install -U pip setuptools wheel twine
- python -m pip install -U -r requirements.txt
+ python devscripts/install_deps.py -o --include build
- name: Prepare
env:
diff --git a/README.md b/README.md
index c74777d2f..2fcb09917 100644
--- a/README.md
+++ b/README.md
@@ -324,7 +324,7 @@ If you do not have the necessary dependencies for a task you are attempting, yt-
To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used. You can run the following commands:
```
-python3 -m pip install -U pyinstaller -r requirements.txt
+python3 devscripts/install_deps.py --include pyinstaller
python3 devscripts/make_lazy_extractors.py
python3 -m bundle.pyinstaller
```
@@ -351,13 +351,14 @@ While we provide the option to build with [py2exe](https://www.py2exe.org), it i
If you wish to build it anyway, install Python (if it is not already installed) and you can run the following commands:
```
-py -m pip install -U py2exe -r requirements.txt
+py devscripts/install_deps.py --include py2exe
py devscripts/make_lazy_extractors.py
py -m bundle.py2exe
```
### Related scripts
+* **`devscripts/install_deps.py`** - Install dependencies for yt-dlp.
* **`devscripts/update-version.py`** - Update the version number based on current date.
* **`devscripts/set-variant.py`** - Set the build variant of the executable.
* **`devscripts/make_changelog.py`** - Create a markdown changelog using short commit messages and update `CONTRIBUTORS` file.
diff --git a/devscripts/install_deps.py b/devscripts/install_deps.py
new file mode 100755
index 000000000..715e5b044
--- /dev/null
+++ b/devscripts/install_deps.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Allow execution from anywhere
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import argparse
+import re
+import subprocess
+
+from devscripts.tomlparse import parse_toml
+from devscripts.utils import read_file
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Install dependencies for yt-dlp')
+ parser.add_argument(
+ 'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)')
+ parser.add_argument(
+ '-e', '--exclude', metavar='REQUIREMENT', action='append', help='Exclude a required dependency')
+ parser.add_argument(
+ '-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group')
+ parser.add_argument(
+ '-o', '--only-optional', action='store_true', help='Only install optional dependencies')
+ parser.add_argument(
+ '-p', '--print', action='store_true', help='Only print a requirements.txt to stdout')
+ parser.add_argument(
+ '-u', '--user', action='store_true', help='Install with pip as --user')
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+ toml_data = parse_toml(read_file(args.input))
+ deps = toml_data['project']['dependencies']
+ targets = deps.copy() if not args.only_optional else []
+
+ for exclude in args.exclude or []:
+ for dep in deps:
+ simplified_dep = re.match(r'[\w-]+', dep)[0]
+ if dep in targets and (exclude.lower() == simplified_dep.lower() or exclude == dep):
+ targets.remove(dep)
+
+ optional_deps = toml_data['project']['optional-dependencies']
+ for include in args.include or []:
+ group = optional_deps.get(include)
+ if group:
+ targets.extend(group)
+
+ if args.print:
+ for target in targets:
+ print(target)
+ return
+
+ pip_args = [sys.executable, '-m', 'pip', 'install', '-U']
+ if args.user:
+ pip_args.append('--user')
+ pip_args.extend(targets)
+
+ return subprocess.call(pip_args)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 06ff82a80..000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-mutagen
-pycryptodomex
-brotli; implementation_name=='cpython'
-brotlicffi; implementation_name!='cpython'
-certifi
-requests>=2.31.0,<3
-urllib3>=1.26.17,<3
-websockets>=12.0
From 920397634d1e84e76d2cb897bd6d69ba0c6bd5ca Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 15:24:41 +0100
Subject: [PATCH 134/248] [build] Fix `secretstorage` for ARM builds
Authored by: bashonly
---
.github/workflows/build.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 082164c9e..0c2b0f684 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -199,10 +199,10 @@ jobs:
dockerRunArgs: --volume "${PWD}/repo:/repo"
install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
apt update
- apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
+ apt -y install zlib1g-dev libffi-dev python3.8 python3.8-dev python3.8-distutils python3-pip
python3.8 -m pip install -U pip setuptools wheel
# Cannot access any files from the repo directory at this stage
- python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
+ python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage cffi
run: |
cd repo
From 867f637b95b342e1cb9f1dc3c6cf0ffe727187ce Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 17:35:27 +0100
Subject: [PATCH 135/248] [cleanup] Build files cleanup
- Fix `AUTHORS` file by doing an unshallow checkout
- Update triggers for nightly/master release
Authored by: bashonly
---
.github/workflows/release-master.yml | 2 ++
.github/workflows/release-nightly.yml | 9 ++++++++-
.github/workflows/release.yml | 2 ++
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 2430dc5f8..a84547580 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -8,6 +8,8 @@ on:
- "!yt_dlp/version.py"
- "bundle/*.py"
- "pyproject.toml"
+ - "Makefile"
+ - ".github/workflows/build.yml"
concurrency:
group: release-master
permissions:
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 16d583846..f459a3a17 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -18,7 +18,14 @@ jobs:
- name: Check for new commits
id: check_for_new_commits
run: |
- relevant_files=("yt_dlp/*.py" ':!yt_dlp/version.py' "bundle/*.py" "pyproject.toml")
+ relevant_files=(
+ "yt_dlp/*.py"
+ ':!yt_dlp/version.py'
+ "bundle/*.py"
+ "pyproject.toml"
+ "Makefile"
+ ".github/workflows/build.yml"
+ )
echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT"
release:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1653add4f..eded11a13 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -246,6 +246,8 @@ jobs:
steps:
- uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: "3.10"
From b14e818b37f62e3224da157b3ad768b3f0815fcd Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 15:47:16 +0100
Subject: [PATCH 136/248] [ci] Bump `actions/setup-python` to v5
Authored by: bashonly
---
.github/workflows/build.yml | 6 +++---
.github/workflows/core.yml | 2 +-
.github/workflows/download.yml | 4 ++--
.github/workflows/quick-test.yml | 4 ++--
.github/workflows/release.yml | 6 +++---
5 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0c2b0f684..4d8e8bf38 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -107,7 +107,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
with:
python-version: "3.10"
- uses: conda-incubator/setup-miniconda@v2
@@ -328,7 +328,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
with: # 3.8 is used for Win7 support
python-version: "3.8"
- name: Install Requirements
@@ -377,7 +377,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
with:
python-version: "3.8"
architecture: "x86"
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index f694c9bdd..ba8630630 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -49,7 +49,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install test requirements
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 84339d970..7256804d9 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -11,7 +11,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: 3.9
- name: Install test requirements
@@ -38,7 +38,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install test requirements
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index 4e9616926..3114e7bdd 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -11,7 +11,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.8
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: '3.8'
- name: Install test requirements
@@ -26,7 +26,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
- name: Install flake8
run: python3 ./devscripts/install_deps.py -o --include dev
- name: Make lazy extractors
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index eded11a13..fac096be7 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -71,7 +71,7 @@ jobs:
with:
fetch-depth: 0
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
with:
python-version: "3.10"
@@ -248,7 +248,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
with:
python-version: "3.10"
@@ -297,7 +297,7 @@ jobs:
with:
fetch-depth: 0
- uses: actions/download-artifact@v3
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
with:
python-version: "3.10"
From b0059f0413a6ba6ab0a3aec1f00188ce083cd8bf Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 15:47:48 +0100
Subject: [PATCH 137/248] [build] Bump `conda-incubator/setup-miniconda` to v3
Authored by: bashonly
---
.github/workflows/build.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4d8e8bf38..e8a97e3f4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -110,7 +110,7 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- - uses: conda-incubator/setup-miniconda@v2
+ - uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
use-mamba: true
From 3876429d72afb35247f4b2531eb9b16cfc7e0968 Mon Sep 17 00:00:00 2001
From: bashonly
Date: Sun, 11 Feb 2024 15:48:09 +0100
Subject: [PATCH 138/248] [build] Bump `actions/upload-artifact` to v4 and
adjust workflows
Authored by: bashonly
---
.github/workflows/build.yml | 36 ++++++++++++++++++++++++++---------
.github/workflows/release.yml | 6 +++++-
2 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e8a97e3f4..cd7ead796 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -162,13 +162,15 @@ jobs:
done
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
+ name: build-${{ github.job }}
path: |
yt-dlp
yt-dlp.tar.gz
yt-dlp_linux
yt-dlp_linux.zip
+ compression-level: 0
linux_arm:
needs: process
@@ -223,10 +225,12 @@ jobs:
fi
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
+ name: build-linux_${{ matrix.architecture }}
path: | # run-on-arch-action designates armv7l as armv7
repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
+ compression-level: 0
macos:
needs: process
@@ -265,11 +269,13 @@ jobs:
[[ "$version" != "$downgraded_version" ]]
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
+ name: build-${{ github.job }}
path: |
dist/yt-dlp_macos
dist/yt-dlp_macos.zip
+ compression-level: 0
macos_legacy:
needs: process
@@ -316,10 +322,12 @@ jobs:
[[ "$version" != "$downgraded_version" ]]
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
+ name: build-${{ github.job }}
path: |
dist/yt-dlp_macos_legacy
+ compression-level: 0
windows:
needs: process
@@ -363,12 +371,14 @@ jobs:
}
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
+ name: build-${{ github.job }}
path: |
dist/yt-dlp.exe
dist/yt-dlp_min.exe
dist/yt-dlp_win.zip
+ compression-level: 0
windows32:
needs: process
@@ -409,10 +419,12 @@ jobs:
}
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
+ name: build-${{ github.job }}
path: |
dist/yt-dlp_x86.exe
+ compression-level: 0
meta_files:
if: inputs.meta_files && always() && !cancelled()
@@ -426,7 +438,11 @@ jobs:
- windows32
runs-on: ubuntu-latest
steps:
- - uses: actions/download-artifact@v3
+ - uses: actions/download-artifact@v4
+ with:
+ path: artifact
+ pattern: build-*
+ merge-multiple: true
- name: Make SHA2-SUMS files
run: |
@@ -461,8 +477,10 @@ jobs:
done
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
+ name: build-${{ github.job }}
path: |
- SHA*SUMS*
_update_spec
+ SHA*SUMS*
+ compression-level: 0
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fac096be7..f5c6a793e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -296,7 +296,11 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- - uses: actions/download-artifact@v3
+ - uses: actions/download-artifact@v4
+ with:
+ path: artifact
+ pattern: build-*
+ merge-multiple: true
- uses: actions/setup-python@v5
with:
python-version: "3.10"
From 1ed5ee2f045f717e814f84ba461dadc58e712266 Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Tue, 13 Feb 2024 04:11:17 +0100
Subject: [PATCH 139/248] [ie/Ant1NewsGrEmbed] Fix extractor (#9191)
Authored by: seproDev
---
yt_dlp/extractor/antenna.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/yt_dlp/extractor/antenna.py b/yt_dlp/extractor/antenna.py
index c78717aa9..17a4b6900 100644
--- a/yt_dlp/extractor/antenna.py
+++ b/yt_dlp/extractor/antenna.py
@@ -78,14 +78,14 @@ class Ant1NewsGrArticleIE(AntennaBaseIE):
_TESTS = [{
'url': 'https://www.ant1news.gr/afieromata/article/549468/o-tzeims-mpont-sta-meteora-oi-apeiles-kai-o-xesikomos-ton-kalogeron',
- 'md5': '294f18331bb516539d72d85a82887dcc',
+ 'md5': '57eb8d12181f0fa2b14b0b138e1de9b6',
'info_dict': {
'id': '_xvg/m_cmbatw=',
'ext': 'mp4',
'title': 'md5:a93e8ecf2e4073bfdffcb38f59945411',
- 'timestamp': 1603092840,
- 'upload_date': '20201019',
- 'thumbnail': 'https://ant1media.azureedge.net/imgHandler/640/756206d2-d640-40e2-b201-3555abdfc0db.jpg',
+ 'timestamp': 1666166520,
+ 'upload_date': '20221019',
+ 'thumbnail': 'https://ant1media.azureedge.net/imgHandler/1920/756206d2-d640-40e2-b201-3555abdfc0db.jpg',
},
}, {
'url': 'https://ant1news.gr/Society/article/620286/symmoria-anilikon-dikigoros-thymaton-ithelan-na-toys-apoteleiosoyn',
@@ -117,7 +117,7 @@ class Ant1NewsGrEmbedIE(AntennaBaseIE):
_BASE_PLAYER_URL_RE = r'(?:https?:)?//(?:[a-zA-Z0-9\-]+\.)?(?:antenna|ant1news)\.gr/templates/pages/player'
_VALID_URL = rf'{_BASE_PLAYER_URL_RE}\?([^#]+&)?cid=(?P[^#&]+)'
_EMBED_REGEX = [rf'