Improved progress reporting (See desc) (#1125)

* Separate `--console-title` and `--no-progress`
* Add option `--progress` to show progress-bar even in quiet mode
* Fix and refactor `minicurses`
* Use `minicurses` for all progress reporting
* Standardize use of terminal sequences and enable color support for windows 10
* Add option `--progress-template` to customize progress-bar and console-title
* Add postprocessor hooks and progress reporting

Closes: #906, #901, #1085, #1170
pull/1326/head
pukkandan 3 years ago committed by GitHub
parent fee3f44f5f
commit 819e05319b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -604,7 +604,18 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
(Alias: --force-download-archive) (Alias: --force-download-archive)
--newline Output progress bar as new lines --newline Output progress bar as new lines
--no-progress Do not print progress bar --no-progress Do not print progress bar
--progress Show progress bar, even if in quiet mode
--console-title Display progress in console titlebar --console-title Display progress in console titlebar
--progress-template [TYPES:]TEMPLATE
Template for progress outputs, optionally
prefixed with one of "download:" (default),
"download-title:" (the console title),
"postprocess:", or "postprocess-title:".
The video's fields are accessible under the
"info" key and the progress attributes are
accessible under "progress" key. Eg:
--console-title --progress-template
"download-title:%(info.id)s-%(progress.eta)s"
-v, --verbose Print various debugging information -v, --verbose Print various debugging information
--dump-pages Print downloaded pages encoded using base64 --dump-pages Print downloaded pages encoded using base64
to debug problems (very verbose) to debug problems (very verbose)

@ -666,8 +666,7 @@ class TestYoutubeDL(unittest.TestCase):
ydl._num_downloads = 1 ydl._num_downloads = 1
self.assertEqual(ydl.validate_outtmpl(tmpl), None) self.assertEqual(ydl.validate_outtmpl(tmpl), None)
outtmpl, tmpl_dict = ydl.prepare_outtmpl(tmpl, info or self.outtmpl_info) out = ydl.evaluate_outtmpl(tmpl, info or self.outtmpl_info)
out = ydl.escape_outtmpl(outtmpl) % tmpl_dict
fname = ydl.prepare_filename(info or self.outtmpl_info) fname = ydl.prepare_filename(info or self.outtmpl_info)
if not isinstance(expected, (list, tuple)): if not isinstance(expected, (list, tuple)):

@ -42,6 +42,7 @@ from .compat import (
compat_urllib_error, compat_urllib_error,
compat_urllib_request, compat_urllib_request,
compat_urllib_request_DataHandler, compat_urllib_request_DataHandler,
windows_enable_vt_mode,
) )
from .cookies import load_cookies from .cookies import load_cookies
from .utils import ( from .utils import (
@ -67,8 +68,6 @@ from .utils import (
float_or_none, float_or_none,
format_bytes, format_bytes,
format_field, format_field,
STR_FORMAT_RE_TMPL,
STR_FORMAT_TYPES,
formatSeconds, formatSeconds,
GeoRestrictedError, GeoRestrictedError,
HEADRequest, HEADRequest,
@ -101,9 +100,13 @@ from .utils import (
sanitize_url, sanitize_url,
sanitized_Request, sanitized_Request,
std_headers, std_headers,
STR_FORMAT_RE_TMPL,
STR_FORMAT_TYPES,
str_or_none, str_or_none,
strftime_or_none, strftime_or_none,
subtitles_filename, subtitles_filename,
supports_terminal_sequences,
TERMINAL_SEQUENCES,
ThrottledDownload, ThrottledDownload,
to_high_limit_path, to_high_limit_path,
traverse_obj, traverse_obj,
@ -248,6 +251,7 @@ class YoutubeDL(object):
rejecttitle: Reject downloads for matching titles. rejecttitle: Reject downloads for matching titles.
logger: Log messages to a logging.Logger instance. logger: Log messages to a logging.Logger instance.
logtostderr: Log messages to stderr instead of stdout. logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
writedescription: Write the video description to a .description file writedescription: Write the video description to a .description file
writeinfojson: Write the video description to a .info.json file writeinfojson: Write the video description to a .info.json file
clean_infojson: Remove private fields from the infojson clean_infojson: Remove private fields from the infojson
@ -353,6 +357,15 @@ class YoutubeDL(object):
Progress hooks are guaranteed to be called at least once Progress hooks are guaranteed to be called at least once
(with status "finished") if the download is successful. (with status "finished") if the download is successful.
postprocessor_hooks: A list of functions that get called on postprocessing
progress, with a dictionary with the entries
* status: One of "started", "processing", or "finished".
Check this first and ignore unknown values.
* postprocessor: Name of the postprocessor
* info_dict: The extracted info_dict
Progress hooks are guaranteed to be called at least twice
(with status "started" and "finished") if the processing is successful.
merge_output_format: Extension to use when merging formats. merge_output_format: Extension to use when merging formats.
final_ext: Expected final extension; used to detect when the file was final_ext: Expected final extension; used to detect when the file was
already downloaded and converted. "merge_output_format" is already downloaded and converted. "merge_output_format" is
@ -412,11 +425,15 @@ class YoutubeDL(object):
filename, abort-on-error, multistreams, no-live-chat, filename, abort-on-error, multistreams, no-live-chat,
no-clean-infojson, no-playlist-metafiles, no-keep-subs. no-clean-infojson, no-playlist-metafiles, no-keep-subs.
Refer __init__.py for their implementation Refer __init__.py for their implementation
progress_template: Dictionary of templates for progress outputs.
Allowed keys are 'download', 'postprocess',
'download-title' (console title) and 'postprocess-title'.
The template is mapped on a dictionary with keys 'progress' and 'info'
The following parameters are not used by YoutubeDL itself, they are used by The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py): the downloader (see yt_dlp/downloader/common.py):
nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize, nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle, max_filesize, test, noresizebuffer, retries, continuedl, noprogress,
xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size. xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size.
The following options are used by the post processors: The following options are used by the post processors:
@ -484,26 +501,27 @@ class YoutubeDL(object):
self._first_webpage_request = True self._first_webpage_request = True
self._post_hooks = [] self._post_hooks = []
self._progress_hooks = [] self._progress_hooks = []
self._postprocessor_hooks = []
self._download_retcode = 0 self._download_retcode = 0
self._num_downloads = 0 self._num_downloads = 0
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self._err_file = sys.stderr self._err_file = sys.stderr
self.params = { self.params = params
# Default parameters
'nocheckcertificate': False,
}
self.params.update(params)
self.cache = Cache(self) self.cache = Cache(self)
windows_enable_vt_mode()
self.params['no_color'] = self.params.get('no_color') or not supports_terminal_sequences(self._err_file)
if sys.version_info < (3, 6): if sys.version_info < (3, 6):
self.report_warning( self.report_warning(
'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys.version_info[:2]) 'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys.version_info[:2])
if self.params.get('allow_unplayable_formats'): if self.params.get('allow_unplayable_formats'):
self.report_warning( self.report_warning(
'You have asked for unplayable formats to be listed/downloaded. ' f'You have asked for {self._color_text("unplayable formats", "blue")} to be listed/downloaded. '
'This is a developer option intended for debugging. ' 'This is a developer option intended for debugging. \n'
'If you experience any issues while using this option, DO NOT open a bug report') ' If you experience any issues while using this option, '
f'{self._color_text("DO NOT", "red")} open a bug report')
def check_deprecated(param, option, suggestion): def check_deprecated(param, option, suggestion):
if self.params.get(param) is not None: if self.params.get(param) is not None:
@ -675,9 +693,13 @@ class YoutubeDL(object):
self._post_hooks.append(ph) self._post_hooks.append(ph)
def add_progress_hook(self, ph): def add_progress_hook(self, ph):
"""Add the progress hook (currently only for the file downloader)""" """Add the download progress hook"""
self._progress_hooks.append(ph) self._progress_hooks.append(ph)
def add_postprocessor_hook(self, ph):
"""Add the postprocessing progress hook"""
self._postprocessor_hooks.append(ph)
def _bidi_workaround(self, message): def _bidi_workaround(self, message):
if not hasattr(self, '_output_channel'): if not hasattr(self, '_output_channel'):
return message return message
@ -790,6 +812,11 @@ class YoutubeDL(object):
self.to_stdout( self.to_stdout(
message, skip_eol, quiet=self.params.get('quiet', False)) message, skip_eol, quiet=self.params.get('quiet', False))
def _color_text(self, text, color):
if self.params.get('no_color'):
return text
return f'{TERMINAL_SEQUENCES[color.upper()]}{text}{TERMINAL_SEQUENCES["RESET_STYLE"]}'
def report_warning(self, message, only_once=False): def report_warning(self, message, only_once=False):
''' '''
Print the message to stderr, it will be prefixed with 'WARNING:' Print the message to stderr, it will be prefixed with 'WARNING:'
@ -800,24 +827,14 @@ class YoutubeDL(object):
else: else:
if self.params.get('no_warnings'): if self.params.get('no_warnings'):
return return
if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt': self.to_stderr(f'{self._color_text("WARNING:", "yellow")} {message}', only_once)
_msg_header = '\033[0;33mWARNING:\033[0m'
else:
_msg_header = 'WARNING:'
warning_message = '%s %s' % (_msg_header, message)
self.to_stderr(warning_message, only_once)
def report_error(self, message, tb=None): def report_error(self, message, tb=None):
''' '''
Do the same as trouble, but prefixes the message with 'ERROR:', colored Do the same as trouble, but prefixes the message with 'ERROR:', colored
in red if stderr is a tty file. in red if stderr is a tty file.
''' '''
if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt': self.trouble(f'{self._color_text("ERROR:", "red")} {message}', tb)
_msg_header = '\033[0;31mERROR:\033[0m'
else:
_msg_header = 'ERROR:'
error_message = '%s %s' % (_msg_header, message)
self.trouble(error_message, tb)
def write_debug(self, message, only_once=False): def write_debug(self, message, only_once=False):
'''Log debug message or Print message to stderr''' '''Log debug message or Print message to stderr'''
@ -919,7 +936,7 @@ class YoutubeDL(object):
return err return err
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None): def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
""" Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """ """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList
@ -1073,6 +1090,10 @@ class YoutubeDL(object):
return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
def evaluate_outtmpl(self, outtmpl, info_dict, *args, **kwargs):
outtmpl, info_dict = self.prepare_outtmpl(outtmpl, info_dict, *args, **kwargs)
return self.escape_outtmpl(outtmpl) % info_dict
def _prepare_filename(self, info_dict, tmpl_type='default'): def _prepare_filename(self, info_dict, tmpl_type='default'):
try: try:
sanitize = lambda k, v: sanitize_filename( sanitize = lambda k, v: sanitize_filename(
@ -2431,10 +2452,8 @@ class YoutubeDL(object):
if self.params.get('forceprint') or self.params.get('forcejson'): if self.params.get('forceprint') or self.params.get('forcejson'):
self.post_extract(info_dict) self.post_extract(info_dict)
for tmpl in self.params.get('forceprint', []): for tmpl in self.params.get('forceprint', []):
if re.match(r'\w+$', tmpl): self.to_stdout(self.evaluate_outtmpl(
tmpl = '%({})s'.format(tmpl) f'%({tmpl})s' if re.match(r'\w+$', tmpl) else tmpl, info_dict))
tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict)
self.to_stdout(self.escape_outtmpl(tmpl) % info_copy)
print_mandatory('title') print_mandatory('title')
print_mandatory('id') print_mandatory('id')

@ -302,11 +302,14 @@ def _real_main(argv=None):
parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err))) parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err)))
for k, tmpl in opts.outtmpl.items(): for k, tmpl in opts.outtmpl.items():
validate_outtmpl(tmpl, '%s output template' % k) validate_outtmpl(tmpl, f'{k} output template')
opts.forceprint = opts.forceprint or [] opts.forceprint = opts.forceprint or []
for tmpl in opts.forceprint or []: for tmpl in opts.forceprint or []:
validate_outtmpl(tmpl, 'print template') validate_outtmpl(tmpl, 'print template')
validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title') validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title')
for k, tmpl in opts.progress_template.items():
k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress'
validate_outtmpl(tmpl, f'{k} template')
if opts.extractaudio and not opts.keepvideo and opts.format is None: if opts.extractaudio and not opts.keepvideo and opts.format is None:
opts.format = 'bestaudio/best' opts.format = 'bestaudio/best'
@ -633,8 +636,9 @@ def _real_main(argv=None):
'noresizebuffer': opts.noresizebuffer, 'noresizebuffer': opts.noresizebuffer,
'http_chunk_size': opts.http_chunk_size, 'http_chunk_size': opts.http_chunk_size,
'continuedl': opts.continue_dl, 'continuedl': opts.continue_dl,
'noprogress': opts.noprogress, 'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress,
'progress_with_newline': opts.progress_with_newline, 'progress_with_newline': opts.progress_with_newline,
'progress_template': opts.progress_template,
'playliststart': opts.playliststart, 'playliststart': opts.playliststart,
'playlistend': opts.playlistend, 'playlistend': opts.playlistend,
'playlistreverse': opts.playlist_reverse, 'playlistreverse': opts.playlist_reverse,

@ -159,6 +159,12 @@ except ImportError:
compat_pycrypto_AES = None compat_pycrypto_AES = None
def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075
if compat_os_name != 'nt':
return
os.system('')
# Deprecated # Deprecated
compat_basestring = str compat_basestring = str
@ -281,5 +287,6 @@ __all__ = [
'compat_xml_parse_error', 'compat_xml_parse_error',
'compat_xpath', 'compat_xpath',
'compat_zip', 'compat_zip',
'windows_enable_vt_mode',
'workaround_optparse_bug9161', 'workaround_optparse_bug9161',
] ]

@ -7,7 +7,6 @@ import sys
import time import time
import random import random
from ..compat import compat_os_name
from ..utils import ( from ..utils import (
decodeArgument, decodeArgument,
encodeFilename, encodeFilename,
@ -17,6 +16,7 @@ from ..utils import (
timeconvert, timeconvert,
) )
from ..minicurses import ( from ..minicurses import (
MultilineLogger,
MultilinePrinter, MultilinePrinter,
QuietMultilinePrinter, QuietMultilinePrinter,
BreaklineStatusPrinter BreaklineStatusPrinter
@ -44,8 +44,6 @@ class FileDownloader(object):
noresizebuffer: Do not automatically resize the download buffer. noresizebuffer: Do not automatically resize the download buffer.
continuedl: Try to continue downloads if possible. continuedl: Try to continue downloads if possible.
noprogress: Do not print the progress bar. noprogress: Do not print the progress bar.
logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
nopart: Do not use temporary .part files. nopart: Do not use temporary .part files.
updatetime: Use the Last-modified header to set output file timestamps. updatetime: Use the Last-modified header to set output file timestamps.
test: Download only first bytes to test the downloader. test: Download only first bytes to test the downloader.
@ -61,6 +59,7 @@ class FileDownloader(object):
http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be
useful for bypassing bandwidth throttling imposed by useful for bypassing bandwidth throttling imposed by
a webserver (experimental) a webserver (experimental)
progress_template: See YoutubeDL.py
Subclasses of this one must re-define the real_download method. Subclasses of this one must re-define the real_download method.
""" """
@ -73,7 +72,7 @@ class FileDownloader(object):
self.ydl = ydl self.ydl = ydl
self._progress_hooks = [] self._progress_hooks = []
self.params = params self.params = params
self._multiline = None self._prepare_multiline_status()
self.add_progress_hook(self.report_progress) self.add_progress_hook(self.report_progress)
@staticmethod @staticmethod
@ -242,43 +241,36 @@ class FileDownloader(object):
"""Report destination filename.""" """Report destination filename."""
self.to_screen('[download] Destination: ' + filename) self.to_screen('[download] Destination: ' + filename)
def _prepare_multiline_status(self, lines): def _prepare_multiline_status(self, lines=1):
if self.params.get('quiet'): if self.params.get('noprogress'):
self._multiline = QuietMultilinePrinter() self._multiline = QuietMultilinePrinter()
elif self.params.get('progress_with_newline', False): elif self.ydl.params.get('logger'):
self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
elif self.params.get('progress_with_newline'):
self._multiline = BreaklineStatusPrinter(sys.stderr, lines) self._multiline = BreaklineStatusPrinter(sys.stderr, lines)
elif self.params.get('noprogress', False):
self._multiline = None
else: else:
self._multiline = MultilinePrinter(sys.stderr, lines) self._multiline = MultilinePrinter(sys.stderr, lines, not self.params.get('quiet'))
def _finish_multiline_status(self): def _finish_multiline_status(self):
if self._multiline is not None:
self._multiline.end() self._multiline.end()
def _report_progress_status(self, msg, is_last_line=False, progress_line=None): def _report_progress_status(self, s):
fullmsg = '[download] ' + msg progress_dict = s.copy()
if self.params.get('progress_with_newline', False): progress_dict.pop('info_dict')
self.to_screen(fullmsg) progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
elif progress_line is not None and self._multiline is not None:
self._multiline.print_at_line(fullmsg, progress_line) progress_template = self.params.get('progress_template', {})
else: self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
if compat_os_name == 'nt' or not sys.stderr.isatty(): progress_template.get('download') or '[download] %(progress._default_template)s',
prev_len = getattr(self, '_report_progress_prev_line_length', 0) progress_dict), s.get('progress_idx') or 0)
if prev_len > len(fullmsg): self.to_console_title(self.ydl.evaluate_outtmpl(
fullmsg += ' ' * (prev_len - len(fullmsg)) progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
self._report_progress_prev_line_length = len(fullmsg) progress_dict))
clear_line = '\r'
else:
clear_line = '\r\x1b[K'
self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
self.to_console_title('yt-dlp ' + msg)
def report_progress(self, s): def report_progress(self, s):
if s['status'] == 'finished': if s['status'] == 'finished':
if self.params.get('noprogress', False): if self.params.get('noprogress'):
self.to_screen('[download] Download completed') self.to_screen('[download] Download completed')
else:
msg_template = '100%%' msg_template = '100%%'
if s.get('total_bytes') is not None: if s.get('total_bytes') is not None:
s['_total_bytes_str'] = format_bytes(s['total_bytes']) s['_total_bytes_str'] = format_bytes(s['total_bytes'])
@ -286,11 +278,9 @@ class FileDownloader(object):
if s.get('elapsed') is not None: if s.get('elapsed') is not None:
s['_elapsed_str'] = self.format_seconds(s['elapsed']) s['_elapsed_str'] = self.format_seconds(s['elapsed'])
msg_template += ' in %(_elapsed_str)s' msg_template += ' in %(_elapsed_str)s'
self._report_progress_status( s['_percent_str'] = self.format_percent(100)
msg_template % s, is_last_line=True, progress_line=s.get('progress_idx')) s['_default_template'] = msg_template % s
return self._report_progress_status(s)
if self.params.get('noprogress'):
return return
if s['status'] != 'downloading': if s['status'] != 'downloading':
@ -332,8 +322,8 @@ class FileDownloader(object):
msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s' msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
else: else:
msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s' msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
s['_default_template'] = msg_template % s
self._report_progress_status(msg_template % s, progress_line=s.get('progress_idx')) self._report_progress_status(s)
def report_resuming_byte(self, resume_len): def report_resuming_byte(self, resume_len):
"""Report attempt to resume at given byte.""" """Report attempt to resume at given byte."""
@ -405,7 +395,9 @@ class FileDownloader(object):
'[download] Sleeping %s seconds ...' % ( '[download] Sleeping %s seconds ...' % (
sleep_interval_sub)) sleep_interval_sub))
time.sleep(sleep_interval_sub) time.sleep(sleep_interval_sub)
return self.real_download(filename, info_dict), True ret = self.real_download(filename, info_dict)
self._finish_multiline_status()
return ret, True
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
"""Real download process. Redefine in subclasses.""" """Real download process. Redefine in subclasses."""

@ -393,9 +393,7 @@ class FragmentFD(FileDownloader):
result = result and job.result() result = result and job.result()
finally: finally:
tpe.shutdown(wait=True) tpe.shutdown(wait=True)
return result
self._finish_multiline_status()
return True
def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None): def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None):
fragment_retries = self.params.get('fragment_retries', 0) fragment_retries = self.params.get('fragment_retries', 0)

@ -1134,10 +1134,7 @@ class InfoExtractor(object):
if mobj: if mobj:
break break
if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty(): _name = self._downloader._color_text(name, 'blue')
_name = '\033[0;34m%s\033[0m' % name
else:
_name = name
if mobj: if mobj:
if group is None: if group is None:

@ -1,10 +1,12 @@
import os
from threading import Lock from threading import Lock
from .utils import compat_os_name, get_windows_version from .utils import supports_terminal_sequences, TERMINAL_SEQUENCES
class MultilinePrinterBase:
def __init__(self, stream=None, lines=1):
self.stream = stream
self.maximum = lines - 1
class MultilinePrinterBase():
def __enter__(self): def __enter__(self):
return self return self
@ -17,119 +19,87 @@ class MultilinePrinterBase():
def end(self): def end(self):
pass pass
def _add_line_number(self, text, line):
if self.maximum:
return f'{line + 1}: {text}'
return text
class MultilinePrinter(MultilinePrinterBase):
def __init__(self, stream, lines): class QuietMultilinePrinter(MultilinePrinterBase):
""" pass
@param stream stream to write to
@lines number of lines to be written
"""
self.stream = stream
is_win10 = compat_os_name == 'nt' and get_windows_version() >= (10, )
self.CARRIAGE_RETURN = '\r'
if os.getenv('TERM') and self._isatty() or is_win10:
# reason not to use curses https://github.com/yt-dlp/yt-dlp/pull/1036#discussion_r713851492
# escape sequences for Win10 https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
self.UP = '\x1b[A'
self.DOWN = '\n'
self.ERASE_LINE = '\x1b[K'
self._HAVE_FULLCAP = self._isatty() or is_win10
else:
self.UP = self.DOWN = self.ERASE_LINE = None
self._HAVE_FULLCAP = False
# lines are numbered from top to bottom, counting from 0 to self.maximum class MultilineLogger(MultilinePrinterBase):
self.maximum = lines - 1 def print_at_line(self, text, pos):
self.lastline = 0 # stream is the logger object, not an actual stream
self.lastlength = 0 self.stream.debug(self._add_line_number(text, pos))
self.movelock = Lock()
@property class BreaklineStatusPrinter(MultilinePrinterBase):
def have_fullcap(self): def print_at_line(self, text, pos):
""" self.stream.write(self._add_line_number(text, pos) + '\n')
True if the TTY is allowing to control cursor,
so that multiline progress works
"""
return self._HAVE_FULLCAP
def _isatty(self):
try:
return self.stream.isatty()
except BaseException:
return False
def _move_cursor(self, dest): class MultilinePrinter(MultilinePrinterBase):
current = min(self.lastline, self.maximum) def __init__(self, stream=None, lines=1, preserve_output=True):
self.stream.write(self.CARRIAGE_RETURN) super().__init__(stream, lines)
if current == dest: self.preserve_output = preserve_output
# current and dest are at same position, no need to move cursor self._lastline = self._lastlength = 0
return self._movelock = Lock()
elif current > dest: self._HAVE_FULLCAP = supports_terminal_sequences(self.stream)
# when maximum == 2,
# 0. dest def lock(func):
# 1. def wrapper(self, *args, **kwargs):
# 2. current with self._movelock:
self.stream.write(self.UP * (current - dest)) return func(self, *args, **kwargs)
elif current < dest: return wrapper
# when maximum == 2,
# 0. current
# 1.
# 2. dest
self.stream.write(self.DOWN * (dest - current))
self.lastline = dest
def _move_cursor(self, dest):
current = min(self._lastline, self.maximum)
self.stream.write('\r')
distance = dest - current
if distance < 0:
self.stream.write(TERMINAL_SEQUENCES['UP'] * -distance)
elif distance > 0:
self.stream.write(TERMINAL_SEQUENCES['DOWN'] * distance)
self._lastline = dest
@lock
def print_at_line(self, text, pos): def print_at_line(self, text, pos):
with self.movelock: if self._HAVE_FULLCAP:
if self.have_fullcap:
self._move_cursor(pos) self._move_cursor(pos)
self.stream.write(self.ERASE_LINE) self.stream.write(TERMINAL_SEQUENCES['ERASE_LINE'])
self.stream.write(text) self.stream.write(text)
else: return
if self.maximum != 0:
# let user know about which line is updating the status text = self._add_line_number(text, pos)
text = f'{pos + 1}: {text}'
textlen = len(text) textlen = len(text)
if self.lastline == pos: if self._lastline == pos:
# move cursor at the start of progress when writing to same line # move cursor at the start of progress when writing to same line
self.stream.write(self.CARRIAGE_RETURN) self.stream.write('\r')
if self.lastlength > textlen: if self._lastlength > textlen:
text += ' ' * (self.lastlength - textlen) text += ' ' * (self._lastlength - textlen)
self.lastlength = textlen self._lastlength = textlen
else: else:
# otherwise, break the line # otherwise, break the line
self.stream.write('\n') self.stream.write('\n')
self.lastlength = 0 self._lastlength = textlen
self.stream.write(text) self.stream.write(text)
self.lastline = pos self._lastline = pos
@lock
def end(self): def end(self):
with self.movelock:
# move cursor to the end of the last line, and write line break # move cursor to the end of the last line, and write line break
# so that other to_screen calls can precede # so that other to_screen calls can precede
if self._HAVE_FULLCAP:
self._move_cursor(self.maximum) self._move_cursor(self.maximum)
if self.preserve_output:
self.stream.write('\n') self.stream.write('\n')
return
if self._HAVE_FULLCAP:
class QuietMultilinePrinter(MultilinePrinterBase): self.stream.write(
def __init__(self): TERMINAL_SEQUENCES['ERASE_LINE']
self.have_fullcap = True + f'{TERMINAL_SEQUENCES["UP"]}{TERMINAL_SEQUENCES["ERASE_LINE"]}' * self.maximum)
else:
self.stream.write(' ' * self._lastlength)
class BreaklineStatusPrinter(MultilinePrinterBase):
def __init__(self, stream, lines):
"""
@param stream stream to write to
"""
self.stream = stream
self.maximum = lines
self.have_fullcap = True
def print_at_line(self, text, pos):
if self.maximum != 0:
# let user know about which line is updating the status
text = f'{pos + 1}: {text}'
self.stream.write(text + '\n')

@ -910,12 +910,30 @@ def parseOpts(overrideArguments=None):
help='Output progress bar as new lines') help='Output progress bar as new lines')
verbosity.add_option( verbosity.add_option(
'--no-progress', '--no-progress',
action='store_true', dest='noprogress', default=False, action='store_true', dest='noprogress', default=None,
help='Do not print progress bar') help='Do not print progress bar')
verbosity.add_option(
'--progress',
action='store_false', dest='noprogress',
help='Show progress bar, even if in quiet mode')
verbosity.add_option( verbosity.add_option(
'--console-title', '--console-title',
action='store_true', dest='consoletitle', default=False, action='store_true', dest='consoletitle', default=False,
help='Display progress in console titlebar') help='Display progress in console titlebar')
verbosity.add_option(
'--progress-template',
metavar='[TYPES:]TEMPLATE', dest='progress_template', default={}, type='str',
action='callback', callback=_dict_from_options_callback,
callback_kwargs={
'allowed_keys': '(download|postprocess)(-title)?',
'default_key': 'download'
}, help=(
'Template for progress outputs, optionally prefixed with one of "download:" (default), '
'"download-title:" (the console title), "postprocess:", or "postprocess-title:". '
'The video\'s fields are accessible under the "info" key and '
'the progress attributes are accessible under "progress" key. Eg: '
# TODO: Document the fields inside "progress"
'--console-title --progress-template "download-title:%(info.id)s-%(progress.eta)s"'))
verbosity.add_option( verbosity.add_option(
'-v', '--verbose', '-v', '--verbose',
action='store_true', dest='verbose', default=False, action='store_true', dest='verbose', default=False,

@ -1,5 +1,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import copy
import functools import functools
import os import os
@ -11,7 +12,26 @@ from ..utils import (
) )
class PostProcessor(object): class PostProcessorMetaClass(type):
@staticmethod
def run_wrapper(func):
@functools.wraps(func)
def run(self, info, *args, **kwargs):
self._hook_progress({'status': 'started'}, info)
ret = func(self, info, *args, **kwargs)
if ret is not None:
_, info = ret
self._hook_progress({'status': 'finished'}, info)
return ret
return run
def __new__(cls, name, bases, attrs):
if 'run' in attrs:
attrs['run'] = cls.run_wrapper(attrs['run'])
return type.__new__(cls, name, bases, attrs)
class PostProcessor(metaclass=PostProcessorMetaClass):
"""Post Processor class. """Post Processor class.
PostProcessor objects can be added to downloaders with their PostProcessor objects can be added to downloaders with their
@ -34,7 +54,9 @@ class PostProcessor(object):
_downloader = None _downloader = None
def __init__(self, downloader=None): def __init__(self, downloader=None):
self._downloader = downloader self._progress_hooks = []
self.add_progress_hook(self.report_progress)
self.set_downloader(downloader)
self.PP_NAME = self.pp_key() self.PP_NAME = self.pp_key()
@classmethod @classmethod
@ -68,6 +90,10 @@ class PostProcessor(object):
def set_downloader(self, downloader): def set_downloader(self, downloader):
"""Sets the downloader for this PP.""" """Sets the downloader for this PP."""
self._downloader = downloader self._downloader = downloader
if not downloader:
return
for ph in downloader._postprocessor_hooks:
self.add_progress_hook(ph)
@staticmethod @staticmethod
def _restrict_to(*, video=True, audio=True, images=True): def _restrict_to(*, video=True, audio=True, images=True):
@ -115,6 +141,39 @@ class PostProcessor(object):
return _configuration_args( return _configuration_args(
self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs) self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs)
def _hook_progress(self, status, info_dict):
if not self._progress_hooks:
return
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
status.update({
'info_dict': copy.deepcopy(info_dict),
'postprocessor': self.pp_key(),
})
for ph in self._progress_hooks:
ph(status)
def add_progress_hook(self, ph):
# See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface
self._progress_hooks.append(ph)
def report_progress(self, s):
s['_default_template'] = '%(postprocessor)s %(status)s' % s
progress_dict = s.copy()
progress_dict.pop('info_dict')
progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
progress_template = self.get_param('progress_template', {})
tmpl = progress_template.get('postprocess')
if tmpl:
self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict))
self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s',
progress_dict))
class AudioConversionError(PostProcessingError): class AudioConversionError(PostProcessingError):
pass pass

@ -62,8 +62,7 @@ class MetadataParserPP(PostProcessor):
def interpretter(self, inp, out): def interpretter(self, inp, out):
def f(info): def f(info):
outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(template, info) data_to_parse = self._downloader.evaluate_outtmpl(template, info)
data_to_parse = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}') self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}')
match = out_re.search(data_to_parse) match = out_re.search(data_to_parse)
if match is None: if match is None:

@ -292,8 +292,7 @@ class ModifyChaptersPP(FFmpegPostProcessor):
'name': SponsorBlockPP.CATEGORIES[category], 'name': SponsorBlockPP.CATEGORIES[category],
'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats] 'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats]
}) })
outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(self._sponsorblock_chapter_title, c) c['title'] = self._downloader.evaluate_outtmpl(self._sponsorblock_chapter_title, c)
c['title'] = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
# Merge identically named sponsors. # Merge identically named sponsors.
if (new_chapters and 'categories' in new_chapters[-1] if (new_chapters and 'categories' in new_chapters[-1]
and new_chapters[-1]['title'] == c['title']): and new_chapters[-1]['title'] == c['title']):

@ -6440,3 +6440,26 @@ def jwt_encode_hs256(payload_data, key, headers={}):
signature_b64 = base64.b64encode(h.digest()) signature_b64 = base64.b64encode(h.digest())
token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64 token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64
return token return token
def supports_terminal_sequences(stream):
if compat_os_name == 'nt':
if get_windows_version() < (10, ):
return False
elif not os.getenv('TERM'):
return False
try:
return stream.isatty()
except BaseException:
return False
TERMINAL_SEQUENCES = {
'DOWN': '\n',
'UP': '\x1b[A',
'ERASE_LINE': '\x1b[K',
'RED': '\033[0;31m',
'YELLOW': '\033[0;33m',
'BLUE': '\033[0;34m',
'RESET_STYLE': '\033[0m',
}

Loading…
Cancel
Save