From 52a8a1e1b93dbc88f0018d4842f1e90ba96e095f Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 10 Apr 2021 20:38:33 +0530 Subject: [PATCH] Option to choose different downloader for different protocols * Renamed `--external-downloader-args` to `--downloader-args` * Added `native` as an option for the downloader * Use similar syntax to `--downloader-args` etc. Eg: `--downloader dash:native --downloader aria2c` * Deprecated `--hls-prefer-native` and `--hls-prefer-ffmpeg` since the same can now be done with `--downloader "m3u8:native"` and `m3u8:ffmpeg` respectively * Split `frag_urls` protocol into `m3u8_frag_urls` and `dash_frag_urls` * Standardize shortening of protocol names with `downloader.shorten_protocol_name` --- README.md | 23 +++++++++++++++-------- yt_dlp/YoutubeDL.py | 24 ++++++++++++++++++------ yt_dlp/downloader/__init__.py | 33 ++++++++++++++++++++++++++++++--- yt_dlp/downloader/dash.py | 4 ++-- yt_dlp/downloader/external.py | 19 +++++++++++++------ yt_dlp/downloader/hls.py | 4 ++-- yt_dlp/options.py | 21 +++++++++++++++------ 7 files changed, 95 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 5c085c6f2..0b238b846 100644 --- a/README.md +++ b/README.md @@ -337,10 +337,6 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t --playlist-random Download playlist videos in random order --xattr-set-filesize Set file xattribute ytdl.filesize with expected file size - --hls-prefer-native Use the native HLS downloader instead of - ffmpeg - --hls-prefer-ffmpeg Use ffmpeg instead of the native HLS - downloader --hls-use-mpegts Use the mpegts container for HLS videos; allowing some players to play the video while downloading, and reducing the chance @@ -350,10 +346,19 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t --no-hls-use-mpegts Do not use the mpegts container for HLS videos. This is default when not downloading live streams - --external-downloader NAME Name or path of the external downloader to - use. Currently supports aria2c, avconv, - axel, curl, ffmpeg, httpie, wget - (Recommended: aria2c) + --downloader [PROTO:]NAME Name or path of the external downloader to + use (optionally) prefixed by the protocols + (http, ftp, m3u8, dash, rstp, rtmp, mms) to + use it for. Currently supports native, + aria2c, avconv, axel, curl, ffmpeg, httpie, + wget (Recommended: aria2c). You can use + this option multiple times to set different + downloaders for different protocols. For + example, --downloader aria2c --downloader + "dash,m3u8:native" will use aria2c for + http/ftp downloads, and the native + downloader for dash/m3u8 downloads + (Alias: --external-downloader) --downloader-args NAME:ARGS Give these arguments to the external downloader. Specify the downloader name and the arguments separated by a colon ":". You @@ -1244,6 +1249,8 @@ These are all the deprecated options and the current alternative to achieve the --metadata-from-title FORMAT --parse-metadata "%(title)s:FORMAT" --prefer-avconv avconv is no longer officially supported (Alias: --no-prefer-ffmpeg) --prefer-ffmpeg Default (Alias: --no-prefer-avconv) + --hls-prefer-native --downloader "m3u8:native" + --hls-prefer-ffmpeg --downloader "m3u8:ffmpeg" --avconv-location avconv is no longer officially supported -C, --call-home Not implemented --no-call-home Default diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 249274fb6..83acf4647 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -111,9 +111,17 @@ from .utils import ( process_communicate_or_kill, ) from .cache import Cache -from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER, _PLUGIN_CLASSES +from .extractor import ( + gen_extractor_classes, + get_info_extractor, + _LAZY_LOADER, + _PLUGIN_CLASSES +) from .extractor.openload import PhantomJSwrapper -from .downloader import get_suitable_downloader +from .downloader import ( + get_suitable_downloader, + shorten_protocol_name +) from .downloader.rtmp import rtmpdump_version from .postprocessor import ( FFmpegFixupM3u8PP, @@ -359,9 +367,13 @@ class YoutubeDL(object): geo_bypass_country The following options determine which downloader is picked: - external_downloader: Executable of the external downloader to call. - None or unset for standard (built-in) downloader. - hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv + external_downloader: A dictionary of protocol keys and the executable of the + external downloader to use for it. The allowed protocols + are default|http|ftp|m3u8|dash|rtsp|rtmp|mms. + Set the value to 'native' to use the native downloader + hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'} + or {'m3u8': 'ffmpeg'} instead. + Use the native HLS downloader instead of ffmpeg/avconv if True, otherwise use ffmpeg/avconv if False, otherwise use downloader suggested by extractor if None. @@ -2776,7 +2788,7 @@ class YoutubeDL(object): '|', format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes), format_field(f, 'tbr', '%4dk'), - f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''), + shorten_protocol_name(f.get('protocol', '').replace("native", "n")), '|', format_field(f, 'vcodec', default='unknown').replace('none', ''), format_field(f, 'vbr', '%4dk'), diff --git a/yt_dlp/downloader/__init__.py b/yt_dlp/downloader/__init__.py index c2e155c0a..ceb075472 100644 --- a/yt_dlp/downloader/__init__.py +++ b/yt_dlp/downloader/__init__.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +from ..compat import compat_str from ..utils import ( determine_protocol, ) @@ -42,6 +43,23 @@ PROTOCOL_MAP = { } +def shorten_protocol_name(proto, simplify=False): + short_protocol_names = { + 'm3u8_native': 'm3u8_n', + 'http_dash_segments': 'dash', + 'niconico_dmc': 'dmc', + } + if simplify: + short_protocol_names.update({ + 'https': 'http', + 'ftps': 'ftp', + 'm3u8_native': 'm3u8', + 'm3u8_frag_urls': 'm3u8', + 'dash_frag_urls': 'dash', + }) + return short_protocol_names.get(proto, proto) + + def get_suitable_downloader(info_dict, params={}, default=HttpFD): """Get the downloader class that can handle the info dict.""" protocol = determine_protocol(info_dict) @@ -50,8 +68,14 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD): # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict): # return FFmpegFD - external_downloader = params.get('external_downloader') - if external_downloader is not None: + downloaders = params.get('external_downloader') + external_downloader = ( + downloaders if isinstance(downloaders, compat_str) + else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default'))) + if external_downloader and external_downloader.lower() == 'native': + external_downloader = 'native' + + if external_downloader not in (None, 'native'): ed = get_external_downloader(external_downloader) if ed.can_download(info_dict, external_downloader): return ed @@ -59,6 +83,8 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD): if protocol.startswith('m3u8'): if info_dict.get('is_live'): return FFmpegFD + elif external_downloader == 'native': + return HlsFD elif _get_real_downloader(info_dict, 'frag_urls', params, None): return HlsFD elif params.get('hls_prefer_native') is True: @@ -70,6 +96,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD): __all__ = [ - 'get_suitable_downloader', 'FileDownloader', + 'get_suitable_downloader', + 'shorten_protocol_name', ] diff --git a/yt_dlp/downloader/dash.py b/yt_dlp/downloader/dash.py index 2f29ff8f2..65dc69f7e 100644 --- a/yt_dlp/downloader/dash.py +++ b/yt_dlp/downloader/dash.py @@ -20,7 +20,7 @@ from ..utils import ( class DashSegmentsFD(FragmentFD): """ Download segments in a DASH manifest. External downloaders can take over - the fragment downloads by supporting the 'frag_urls' protocol + the fragment downloads by supporting the 'dash_frag_urls' protocol """ FD_NAME = 'dashsegments' @@ -30,7 +30,7 @@ class DashSegmentsFD(FragmentFD): fragments = info_dict['fragments'][:1] if self.params.get( 'test', False) else info_dict['fragments'] - real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None) + real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None) ctx = { 'filename': filename, diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 63e430ac9..ea2e6eb12 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -81,11 +81,15 @@ class ExternalFD(FileDownloader): @property def exe(self): - return self.params.get('external_downloader') + return self.get_basename() @classmethod def available(cls, path=None): - return check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT]) + path = check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT]) + if path: + cls.exe = path + return path + return False @classmethod def supports(cls, info_dict): @@ -259,7 +263,7 @@ class WgetFD(ExternalFD): class Aria2cFD(ExternalFD): AVAILABLE_OPT = '-v' - SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'frag_urls') + SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls') @staticmethod def supports_manifest(manifest): @@ -310,9 +314,11 @@ class Aria2cFD(ExternalFD): class HttpieFD(ExternalFD): + AVAILABLE_OPT = '--version' + @classmethod def available(cls, path=None): - return check_executable(path or 'http', ['--version']) + return ExternalFD.available(cls, path or 'http') def _make_cmd(self, tmpfilename, info_dict): cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']] @@ -327,7 +333,8 @@ class FFmpegFD(ExternalFD): SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms') @classmethod - def available(cls, path=None): # path is ignored for ffmpeg + def available(cls, path=None): + # TODO: Fix path for ffmpeg return FFmpegPostProcessor().available def _call_downloader(self, tmpfilename, info_dict): @@ -484,4 +491,4 @@ def get_external_downloader(external_downloader): downloader . """ # Drop .exe extension on Windows bn = os.path.splitext(os.path.basename(external_downloader))[0] - return _BY_NAME[bn] + return _BY_NAME.get(bn) diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py index d09bfa3aa..67b09144d 100644 --- a/yt_dlp/downloader/hls.py +++ b/yt_dlp/downloader/hls.py @@ -32,7 +32,7 @@ from ..utils import ( class HlsFD(FragmentFD): """ Download segments in a m3u8 manifest. External downloaders can take over - the fragment downloads by supporting the 'frag_urls' protocol and + the fragment downloads by supporting the 'm3u8_frag_urls' protocol and re-defining 'supports_manifest' function """ @@ -95,7 +95,7 @@ class HlsFD(FragmentFD): # fd.add_progress_hook(ph) return fd.real_download(filename, info_dict) - real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None) + real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None) if real_downloader and not real_downloader.supports_manifest(s): real_downloader = None if real_downloader: diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 6b4736e97..7f36777d9 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -639,11 +639,11 @@ def parseOpts(overrideArguments=None): downloader.add_option( '--hls-prefer-native', dest='hls_prefer_native', action='store_true', default=None, - help='Use the native HLS downloader instead of ffmpeg') + help=optparse.SUPPRESS_HELP) downloader.add_option( '--hls-prefer-ffmpeg', dest='hls_prefer_native', action='store_false', default=None, - help='Use ffmpeg instead of the native HLS downloader') + help=optparse.SUPPRESS_HELP) downloader.add_option( '--hls-use-mpegts', dest='hls_use_mpegts', action='store_true', default=None, @@ -659,11 +659,20 @@ def parseOpts(overrideArguments=None): 'Do not use the mpegts container for HLS videos. ' 'This is default when not downloading live streams')) downloader.add_option( - '--external-downloader', - dest='external_downloader', metavar='NAME', + '--downloader', '--external-downloader', + dest='external_downloader', metavar='[PROTO:]NAME', default={}, type='str', + action='callback', callback=_dict_from_multiple_values_options_callback, + callback_kwargs={ + 'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms', + 'default_key': 'default', 'process': lambda x: x.strip()}, help=( - 'Name or path of the external downloader to use. ' - 'Currently supports %s (Recommended: aria2c)' % ', '.join(list_external_downloaders()))) + 'Name or path of the external downloader to use (optionally) prefixed by ' + 'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. ' + 'Currently supports native, %s (Recommended: aria2c). ' + 'You can use this option multiple times to set different downloaders for different protocols. ' + 'For example, --downloader aria2c --downloader "dash,m3u8:native" will use ' + 'aria2c for http/ftp downloads, and the native downloader for dash/m3u8 downloads ' + '(Alias: --external-downloader)' % ', '.join(list_external_downloaders()))) downloader.add_option( '--downloader-args', '--external-downloader-args', metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',