From 53ed7066ab711ca9a167174de0b61eeed2d04fa4 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 11 May 2021 13:30:48 +0530 Subject: [PATCH] Option `--compat-options` to revert some of yt-dlp's changes * Deprecates `--list-formats-as-table`, `--list-formats-old` --- README.md | 35 +++++++++++++++--- yt_dlp/YoutubeDL.py | 28 ++++++++++----- yt_dlp/__init__.py | 71 +++++++++++++++++++++++++++++++++++-- yt_dlp/extractor/common.py | 9 ++++- yt_dlp/extractor/youtube.py | 6 ++-- yt_dlp/options.py | 20 +++++++---- 6 files changed, 145 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 30673d704..68392c92a 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ A command-line program to download videos from YouTube and many other [video pla yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on the now inactive [youtube-dlc](https://github.com/blackjack4494/yt-dlc). The main focus of this project is adding new features and patches while also keeping up to date with the original project * [NEW FEATURES](#new-features) + * [Differences in default behavior](#differences-in-default-behavior) * [INSTALLATION](#installation) * [Dependencies](#dependencies) * [Update](#update) @@ -105,6 +106,29 @@ See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/comm If you are coming from [youtube-dl](https://github.com/ytdl-org/youtube-dl), the amount of changes are very large. Compare [options](#options) and [supported sites](supportedsites.md) with youtube-dl's to get an idea of the massive number of features/patches [youtube-dlc](https://github.com/blackjack4494/yt-dlc) has accumulated. +### Differences in default behavior + +Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc. + +1. The options `--id`, `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details +1. `avconv` is not supported as as an alternative to `ffmpeg` +1. The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s.%(id)s.%(ext)s`. Instead, you may use `--compat-options filename` +1. The default [format sorting](sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order +1. The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be prefered. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this +1. Unlike youtube-dlc, yt-dlp does not allow merging multiple audio/video streams into one file by default (since this conflicts with the use of `-f bv*+ba`). If needed, this feature must be enabled using `--audio-multistreams` and `--video-multistreams`. You can also use `--compat-options multistreams` to enable both +1. `--ignore-errors` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead +1. When writing metadata files such as thumbnails, description or infojson, the same information (if available) is also written for playlists. Use `--no-write-playlist-metafiles` or `--compat-options no-playlist-metafiles` to not write these files +1. `playlist_index` behaves differently when used with options like `--playlist-reverse` and `--playlist-items`. See [#302](https://github.com/yt-dlp/yt-dlp/issues/302) for details. You can use `--compat-options playlist-index` if you want to keep the earlier behavior +1. The output of `-F` is listed in a new format. Use `--compat-options list-formats` to revert this +1. Youtube live chat (if available) is considered as a subtitle. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent live chat from downloading +1. Youtube channel URLs are automatically redirected to `/video`. Either append a `/featured` to the URL or use `--compat-options no-youtube-channel-redirect` to download only the videos in the home page +1. Unavailable videos are also listed for youtube playlists. Use `--compat-options no-youtube-unavailable-videos` to remove this + +For ease of use, a few more compat options are available: +1. `--compat-options all` = Use all compat options +1. `--compat-options youtube-dl` = `--compat-options all,-multistreams` +1. `--compat-options youtube-dlc` = `--compat-options all,-no-live-chat,-no-youtube-channel-redirect` + # INSTALLATION yt-dlp is not platform specific. So it should work on your Unix box, on Windows or on macOS @@ -212,6 +236,11 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t --mark-watched Mark videos watched (YouTube only) --no-mark-watched Do not mark videos watched (default) --no-colors Do not emit color codes in output + --compat-options OPTS Options that can help keep compatibility + with youtube-dl and youtube-dlc + configurations by reverting some of the + changes made in yt-dlp. See "Differences in + default behavior" for details ## Network Options: --proxy URL Use the specified HTTP/HTTPS/SOCKS proxy. @@ -583,10 +612,6 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t actually downloadable (Experimental) -F, --list-formats List all available formats of requested videos - --list-formats-as-table Present the output of -F in tabular form - (default) - --list-formats-old Present the output of -F in the old form - (Alias: --no-list-formats-as-table) --merge-output-format FORMAT If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, @@ -1286,6 +1311,8 @@ While these options still work, their use is not recommended since there are oth --metadata-from-title FORMAT --parse-metadata "%(title)s:FORMAT" --hls-prefer-native --downloader "m3u8:native" --hls-prefer-ffmpeg --downloader "m3u8:ffmpeg" + --list-formats-old --compat-options list-formats (Alias: --no-list-formats-as-table) + --list-formats-as-table --compat-options -list-formats [Default] (Alias: --no-list-formats-old) --sponskrub-args ARGS --ppa "sponskrub:ARGS" --test Used by developers for testing extractors. Not intended for the end user diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 0111246ca..3cf86cee7 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -385,6 +385,10 @@ class YoutubeDL(object): Use the native HLS downloader instead of ffmpeg/avconv if True, otherwise use ffmpeg/avconv if False, otherwise use downloader suggested by extractor if None. + compat_opts: Compatibility options. See "Differences in default behavior". + Note that only format-sort, format-spec, no-live-chat, + playlist-index, list-formats, no-youtube-channel-redirect + and no-youtube-unavailable-videos works when used via the API The following parameters are not used by YoutubeDL itself, they are used by the downloader (see yt_dlp/downloader/common.py): @@ -470,8 +474,7 @@ class YoutubeDL(object): def check_deprecated(param, option, suggestion): if self.params.get(param) is not None: - self.report_warning( - '%s is deprecated. Use %s instead' % (option, suggestion)) + self.report_warning('%s is deprecated. Use %s instead' % (option, suggestion)) return True return False @@ -479,9 +482,9 @@ class YoutubeDL(object): if self.params.get('geo_verification_proxy') is None: self.params['geo_verification_proxy'] = self.params['cn_verification_proxy'] - check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits') check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"') check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"') + check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"') for msg in self.params.get('warnings', []): self.report_warning(msg) @@ -1401,6 +1404,8 @@ class YoutubeDL(object): max_failures = self.params.get('skip_playlist_after_errors') or float('inf') for i, entry_tuple in enumerate(entries, 1): playlist_index, entry = entry_tuple + if 'playlist_index' in self.params.get('compat_options', []): + playlist_index = playlistitems[i - 1] if playlistitems else i self.to_screen('[download] Downloading video %s of %s' % (i, n_entries)) # This __x_forwarded_for_ip thing is a bit ugly but requires # minimal changes @@ -1519,12 +1524,14 @@ class YoutubeDL(object): not can_merge() or info_dict.get('is_live', False) or self.outtmpl_dict['default'] == '-')) + compat = ( + prefer_best + or self.params.get('allow_multiple_audio_streams', False) + or 'format-spec' in self.params.get('compat_opts', [])) return ( - 'best/bestvideo+bestaudio' - if prefer_best - else 'bestvideo*+bestaudio/best' - if not self.params.get('allow_multiple_audio_streams', False) + 'best/bestvideo+bestaudio' if prefer_best + else 'bestvideo*+bestaudio/best' if not compat else 'bestvideo+bestaudio/best') def build_format_selector(self, format_spec): @@ -2913,7 +2920,9 @@ class YoutubeDL(object): def list_formats(self, info_dict): formats = info_dict.get('formats', [info_dict]) - new_format = self.params.get('listformats_table', False) + new_format = ( + 'list-formats' not in self.params.get('compat_opts', []) + and self.params.get('list_formats_as_table', True) is not False) if new_format: table = [ [ @@ -3014,6 +3023,9 @@ class YoutubeDL(object): if _PLUGIN_CLASSES: self._write_string( '[debug] Plugin Extractors: %s\n' % [ie.ie_key() for ie in _PLUGIN_CLASSES]) + if self.params.get('compat_opts'): + self._write_string( + '[debug] Compatibility options: %s\n' % ', '.join(self.params.get('compat_opts'))) try: sp = subprocess.Popen( ['git', 'rev-parse', '--short', 'HEAD'], diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 6cc4f2f8c..356772b1d 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -235,11 +235,75 @@ def _real_main(argv=None): else: date = DateRange(opts.dateafter, opts.datebefore) - # Do not download videos when there are audio-only formats + def parse_compat_opts(): + parsed_compat_opts, compat_opts = set(), opts.compat_opts[::-1] + while compat_opts: + actual_opt = opt = compat_opts.pop().lower() + if opt == 'youtube-dl': + compat_opts.extend(['-multistreams', 'all']) + elif opt == 'youtube-dlc': + compat_opts.extend(['-no-youtube-channel-redirect', '-no-live-chat', 'all']) + elif opt == 'all': + parsed_compat_opts.update(all_compat_opts) + elif opt == '-all': + parsed_compat_opts = set() + else: + if opt[0] == '-': + opt = opt[1:] + parsed_compat_opts.discard(opt) + else: + parsed_compat_opts.update([opt]) + if opt not in all_compat_opts: + parser.error('Invalid compatibility option %s' % actual_opt) + return parsed_compat_opts + + all_compat_opts = [ + 'filename', 'format-sort', 'abort-on-error', 'format-spec', 'multistreams', + 'no-playlist-metafiles', 'no-live-chat', 'playlist-index', 'list-formats', + 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', + ] + compat_opts = parse_compat_opts() + + def _unused_compat_opt(name): + if name not in compat_opts: + return False + compat_opts.discard(name) + compat_opts.update(['*%s' % name]) + return True + + def set_default_compat(compat_name, opt_name, default=True, remove_compat=False): + attr = getattr(opts, opt_name) + if compat_name in compat_opts: + if attr is None: + setattr(opts, opt_name, not default) + return True + else: + if remove_compat: + _unused_compat_opt(compat_name) + return False + elif attr is None: + setattr(opts, opt_name, default) + return None + + set_default_compat('abort-on-error', 'ignoreerrors') + set_default_compat('no-playlist-metafiles', 'allow_playlist_files') + if 'format-sort' in compat_opts: + opts.format_sort.extend(InfoExtractor.FormatSort.ytdl_default) + _video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False) + _audio_multistreams_set = set_default_compat('multistreams', 'allow_multiple_audio_streams', False, remove_compat=False) + if _video_multistreams_set is False and _audio_multistreams_set is False: + _unused_compat_opt('multistreams') + outtmpl_default = opts.outtmpl.get('default') + if 'filename' in compat_opts: + if outtmpl_default is None: + outtmpl_default = '%(title)s.%(id)s.%(ext)s' + opts.outtmpl.update({'default': outtmpl_default}) + else: + _unused_compat_opt('filename') + if opts.extractaudio and not opts.keepvideo and opts.format is None: opts.format = 'bestaudio/best' - outtmpl_default = opts.outtmpl.get('default') if outtmpl_default is not None and not os.path.splitext(outtmpl_default)[1] and opts.extractaudio: parser.error('Cannot download a video and extract audio into the same' ' file! Use "{0}.%(ext)s" instead of "{0}" as the output' @@ -574,8 +638,9 @@ def _real_main(argv=None): 'geo_bypass': opts.geo_bypass, 'geo_bypass_country': opts.geo_bypass_country, 'geo_bypass_ip_block': opts.geo_bypass_ip_block, - # just for deprecation check 'warnings': warnings, + 'compat_opts': compat_opts, + # just for deprecation check 'autonumber': opts.autonumber or None, 'usetitle': opts.usetitle or None, 'useid': opts.useid or None, diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index e2a9a3801..0112585af 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -557,6 +557,10 @@ class InfoExtractor(object): ie_result = self._real_extract(url) if self._x_forwarded_for_ip: ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip + subtitles = ie_result.get('subtitles') + if (subtitles and 'live_chat' in subtitles + and 'no-live-chat' in self._downloader.params.get('compat_opts')): + del subtitles['live_chat'] return ie_result except GeoRestrictedError as e: if self.__maybe_fake_ip_and_retry(e.countries): @@ -1415,7 +1419,10 @@ class InfoExtractor(object): default = ('hidden', 'hasvid', 'ie_pref', 'lang', 'quality', 'res', 'fps', 'codec:vp9.2', 'size', 'br', 'asr', - 'proto', 'ext', 'has_audio', 'source', 'format_id') # These must not be aliases + 'proto', 'ext', 'hasaud', 'source', 'format_id') # These must not be aliases + ytdl_default = ('hasaud', 'quality', 'tbr', 'filesize', 'vbr', + 'height', 'width', 'proto', 'vext', 'abr', 'aext', + 'fps', 'fs_approx', 'source', 'format_id') settings = { 'vcodec': {'type': 'ordered', 'regex': True, diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index d68970f30..71eb018e6 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -3481,11 +3481,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): item_id = self._match_id(url) url = compat_urlparse.urlunparse( compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com')) + compat_opts = self._downloader.params.get('compat_opts', []) # This is not matched in a channel page with a tab selected mobj = re.match(r'(?P
%s)(?P/?(?![^#?]).*$)' % self._VALID_URL, url)
         mobj = mobj.groupdict() if mobj else {}
-        if mobj and not mobj.get('not_channel'):
+        if mobj and not mobj.get('not_channel') and 'no-youtube-channel-redirect' not in compat_opts:
             self.report_warning(
                 'A channel/user page was given. All the channel\'s videos will be downloaded. '
                 'To download only the videos in the home page, add a "/featured" to the URL')
@@ -3513,7 +3514,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         webpage, data = self._extract_webpage(url, item_id)
 
         # YouTube sometimes provides a button to reload playlist with unavailable videos.
-        data = self._reload_with_unavailable_videos(item_id, data, webpage) or data
+        if 'no-youtube-unavailable-videos' not in compat_opts:
+            data = self._reload_with_unavailable_videos(item_id, data, webpage) or data
 
         tabs = try_get(
             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 3c103f6da..49c3f7d63 100644
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -165,7 +165,7 @@ def parseOpts(overrideArguments=None):
         help='Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
     general.add_option(
         '-i', '--ignore-errors', '--no-abort-on-error',
-        action='store_true', dest='ignoreerrors', default=True,
+        action='store_true', dest='ignoreerrors', default=None,
         help='Continue on download errors, for example to skip unavailable videos in a playlist (default) (Alias: --no-abort-on-error)')
     general.add_option(
         '--abort-on-error', '--no-ignore-errors',
@@ -229,6 +229,14 @@ def parseOpts(overrideArguments=None):
         '--no-colors',
         action='store_true', dest='no_color', default=False,
         help='Do not emit color codes in output')
+    general.add_option(
+        '--compat-options',
+        metavar='OPTS', dest='compat_opts', default=[],
+        action='callback', callback=_comma_separated_values_options_callback, type='str',
+        help=(
+            'Options that can help keep compatibility with youtube-dl and youtube-dlc '
+            'configurations by reverting some of the changes made in yt-dlp. '
+            'See "Differences in default behavior" for details'))
 
     network = optparse.OptionGroup(parser, 'Network Options')
     network.add_option(
@@ -474,7 +482,7 @@ def parseOpts(overrideArguments=None):
             'see "Sorting Formats" for more details'))
     video_format.add_option(
         '--video-multistreams',
-        action='store_true', dest='allow_multiple_video_streams', default=False,
+        action='store_true', dest='allow_multiple_video_streams', default=None,
         help='Allow multiple video streams to be merged into a single file')
     video_format.add_option(
         '--no-video-multistreams',
@@ -482,7 +490,7 @@ def parseOpts(overrideArguments=None):
         help='Only one video stream is downloaded for each output file (default)')
     video_format.add_option(
         '--audio-multistreams',
-        action='store_true', dest='allow_multiple_audio_streams', default=False,
+        action='store_true', dest='allow_multiple_audio_streams', default=None,
         help='Allow multiple audio streams to be merged into a single file')
     video_format.add_option(
         '--no-audio-multistreams',
@@ -513,11 +521,11 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--list-formats-as-table',
         action='store_true', dest='listformats_table', default=True,
-        help='Present the output of -F in tabular form (default)')
+        help=optparse.SUPPRESS_HELP)
     video_format.add_option(
         '--list-formats-old', '--no-list-formats-as-table',
         action='store_false', dest='listformats_table',
-        help='Present the output of -F in the old form (Alias: --no-list-formats-as-table)')
+        help=optparse.SUPPRESS_HELP)
     video_format.add_option(
         '--merge-output-format',
         action='store', dest='merge_output_format', metavar='FORMAT', default=None,
@@ -1012,7 +1020,7 @@ def parseOpts(overrideArguments=None):
         help='Do not write video annotations (default)')
     filesystem.add_option(
         '--write-playlist-metafiles',
-        action='store_true', dest='allow_playlist_files', default=True,
+        action='store_true', dest='allow_playlist_files', default=None,
         help=(
             'Write playlist metadata in addition to the video metadata '
             'when using --write-info-json, --write-description etc. (default)'))