Improved progress reporting (See desc) (#1125)

* Separate `--console-title` and `--no-progress`
* Add option `--progress` to show progress-bar even in quiet mode
* Fix and refactor `minicurses`
* Use `minicurses` for all progress reporting
* Standardize use of terminal sequences and enable color support for windows 10
* Add option `--progress-template` to customize progress-bar and console-title
* Add postprocessor hooks and progress reporting

Closes: #906, #901, #1085, #1170

README.md

@@ -604,7 +604,18 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      (Alias: --force-download-archive)
     --newline                        Output progress bar as new lines
     --no-progress                    Do not print progress bar
+    --progress                       Show progress bar, even if in quiet mode
     --console-title                  Display progress in console titlebar
+    --progress-template [TYPES:]TEMPLATE
+                                     Template for progress outputs, optionally
+                                     prefixed with one of "download:" (default),
+                                     "download-title:" (the console title),
+                                     "postprocess:",  or "postprocess-title:".
+                                     The video's fields are accessible under the
+                                     "info" key and the progress attributes are
+                                     accessible under "progress" key. Eg:
+                                     --console-title --progress-template
+                                     "download-title:%(info.id)s-%(progress.eta)s"
     -v, --verbose                    Print various debugging information
     --dump-pages                     Print downloaded pages encoded using base64
                                      to debug problems (very verbose)

test/test_YoutubeDL.py

@@ -666,8 +666,7 @@ class TestYoutubeDL(unittest.TestCase):
             ydl._num_downloads = 1
             self.assertEqual(ydl.validate_outtmpl(tmpl), None)
 
-            outtmpl, tmpl_dict = ydl.prepare_outtmpl(tmpl, info or self.outtmpl_info)
+            out = ydl.evaluate_outtmpl(tmpl, info or self.outtmpl_info)
-            out = ydl.escape_outtmpl(outtmpl) % tmpl_dict
             fname = ydl.prepare_filename(info or self.outtmpl_info)
 
             if not isinstance(expected, (list, tuple)):

yt_dlp/YoutubeDL.py

@@ -42,6 +42,7 @@ from .compat import (
     compat_urllib_error,
     compat_urllib_request,
     compat_urllib_request_DataHandler,
+    windows_enable_vt_mode,
 )
 from .cookies import load_cookies
 from .utils import (
@@ -67,8 +68,6 @@ from .utils import (
     float_or_none,
     format_bytes,
     format_field,
-    STR_FORMAT_RE_TMPL,
-    STR_FORMAT_TYPES,
     formatSeconds,
     GeoRestrictedError,
     HEADRequest,
@@ -101,9 +100,13 @@ from .utils import (
     sanitize_url,
     sanitized_Request,
     std_headers,
+    STR_FORMAT_RE_TMPL,
+    STR_FORMAT_TYPES,
     str_or_none,
     strftime_or_none,
     subtitles_filename,
+    supports_terminal_sequences,
+    TERMINAL_SEQUENCES,
     ThrottledDownload,
     to_high_limit_path,
     traverse_obj,
@@ -248,6 +251,7 @@ class YoutubeDL(object):
     rejecttitle:       Reject downloads for matching titles.
     logger:            Log messages to a logging.Logger instance.
     logtostderr:       Log messages to stderr instead of stdout.
+    consoletitle:       Display progress in console window's titlebar.
     writedescription:  Write the video description to a .description file
     writeinfojson:     Write the video description to a .info.json file
     clean_infojson:    Remove private fields from the infojson
@@ -353,6 +357,15 @@ class YoutubeDL(object):
 
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
+    postprocessor_hooks:  A list of functions that get called on postprocessing
+                       progress, with a dictionary with the entries
+                       * status: One of "started", "processing", or "finished".
+                                 Check this first and ignore unknown values.
+                       * postprocessor: Name of the postprocessor
+                       * info_dict: The extracted info_dict
+
+                       Progress hooks are guaranteed to be called at least twice
+                       (with status "started" and "finished") if the processing is successful.
     merge_output_format: Extension to use when merging formats.
     final_ext:         Expected final extension; used to detect when the file was
                        already downloaded and converted. "merge_output_format" is
@@ -412,11 +425,15 @@ class YoutubeDL(object):
                        filename, abort-on-error, multistreams, no-live-chat,
                        no-clean-infojson, no-playlist-metafiles, no-keep-subs.
                        Refer __init__.py for their implementation
+    progress_template: Dictionary of templates for progress outputs.
+                       Allowed keys are 'download', 'postprocess',
+                       'download-title' (console title) and 'postprocess-title'.
+                       The template is mapped on a dictionary with keys 'progress' and 'info'
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the downloader (see yt_dlp/downloader/common.py):
     nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
-    max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle,
+    max_filesize, test, noresizebuffer, retries, continuedl, noprogress,
     xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size.
 
     The following options are used by the post processors:
@@ -484,26 +501,27 @@ class YoutubeDL(object):
         self._first_webpage_request = True
         self._post_hooks = []
         self._progress_hooks = []
+        self._postprocessor_hooks = []
         self._download_retcode = 0
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
-        self.params = {
+        self.params = params
-            # Default parameters
-            'nocheckcertificate': False,
-        }
-        self.params.update(params)
         self.cache = Cache(self)
 
+        windows_enable_vt_mode()
+        self.params['no_color'] = self.params.get('no_color') or not supports_terminal_sequences(self._err_file)
+
         if sys.version_info < (3, 6):
             self.report_warning(
                 'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys.version_info[:2])
 
         if self.params.get('allow_unplayable_formats'):
             self.report_warning(
-                'You have asked for unplayable formats to be listed/downloaded. '
+                f'You have asked for {self._color_text("unplayable formats", "blue")} to be listed/downloaded. '
-                'This is a developer option intended for debugging. '
+                'This is a developer option intended for debugging. \n'
-                'If you experience any issues while using this option, DO NOT open a bug report')
+                '         If you experience any issues while using this option, '
+                f'{self._color_text("DO NOT", "red")} open a bug report')
 
         def check_deprecated(param, option, suggestion):
             if self.params.get(param) is not None:
@@ -675,9 +693,13 @@ class YoutubeDL(object):
         self._post_hooks.append(ph)
 
     def add_progress_hook(self, ph):
-        """Add the progress hook (currently only for the file downloader)"""
+        """Add the download progress hook"""
         self._progress_hooks.append(ph)
 
+    def add_postprocessor_hook(self, ph):
+        """Add the postprocessing progress hook"""
+        self._postprocessor_hooks.append(ph)
+
     def _bidi_workaround(self, message):
         if not hasattr(self, '_output_channel'):
             return message
@@ -790,6 +812,11 @@ class YoutubeDL(object):
         self.to_stdout(
             message, skip_eol, quiet=self.params.get('quiet', False))
 
+    def _color_text(self, text, color):
+        if self.params.get('no_color'):
+            return text
+        return f'{TERMINAL_SEQUENCES[color.upper()]}{text}{TERMINAL_SEQUENCES["RESET_STYLE"]}'
+
     def report_warning(self, message, only_once=False):
         '''
         Print the message to stderr, it will be prefixed with 'WARNING:'
@@ -800,24 +827,14 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
+            self.to_stderr(f'{self._color_text("WARNING:", "yellow")} {message}', only_once)
-                _msg_header = '\033[0;33mWARNING:\033[0m'
-            else:
-                _msg_header = 'WARNING:'
-            warning_message = '%s %s' % (_msg_header, message)
-            self.to_stderr(warning_message, only_once)
 
     def report_error(self, message, tb=None):
         '''
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
+        self.trouble(f'{self._color_text("ERROR:", "red")} {message}', tb)
-            _msg_header = '\033[0;31mERROR:\033[0m'
-        else:
-            _msg_header = 'ERROR:'
-        error_message = '%s %s' % (_msg_header, message)
-        self.trouble(error_message, tb)
 
     def write_debug(self, message, only_once=False):
         '''Log debug message or Print message to stderr'''
@@ -919,7 +936,7 @@ class YoutubeDL(object):
             return err
 
     def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
-        """ Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """
+        """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
         info_dict.setdefault('epoch', int(time.time()))  # keep epoch consistent once set
 
         info_dict = dict(info_dict)  # Do not sanitize so as not to consume LazyList
@@ -1073,6 +1090,10 @@ class YoutubeDL(object):
 
         return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
 
+    def evaluate_outtmpl(self, outtmpl, info_dict, *args, **kwargs):
+        outtmpl, info_dict = self.prepare_outtmpl(outtmpl, info_dict, *args, **kwargs)
+        return self.escape_outtmpl(outtmpl) % info_dict
+
     def _prepare_filename(self, info_dict, tmpl_type='default'):
         try:
             sanitize = lambda k, v: sanitize_filename(
@@ -2431,10 +2452,8 @@ class YoutubeDL(object):
         if self.params.get('forceprint') or self.params.get('forcejson'):
             self.post_extract(info_dict)
         for tmpl in self.params.get('forceprint', []):
-            if re.match(r'\w+$', tmpl):
+            self.to_stdout(self.evaluate_outtmpl(
-                tmpl = '%({})s'.format(tmpl)
+                f'%({tmpl})s' if re.match(r'\w+$', tmpl) else tmpl, info_dict))
-            tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict)
-            self.to_stdout(self.escape_outtmpl(tmpl) % info_copy)
 
         print_mandatory('title')
         print_mandatory('id')

yt_dlp/__init__.py

@@ -302,11 +302,14 @@ def _real_main(argv=None):
             parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err)))
 
     for k, tmpl in opts.outtmpl.items():
-        validate_outtmpl(tmpl, '%s output template' % k)
+        validate_outtmpl(tmpl, f'{k} output template')
     opts.forceprint = opts.forceprint or []
     for tmpl in opts.forceprint or []:
         validate_outtmpl(tmpl, 'print template')
     validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title')
+    for k, tmpl in opts.progress_template.items():
+        k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress'
+        validate_outtmpl(tmpl, f'{k} template')
 
     if opts.extractaudio and not opts.keepvideo and opts.format is None:
         opts.format = 'bestaudio/best'
@@ -633,8 +636,9 @@ def _real_main(argv=None):
         'noresizebuffer': opts.noresizebuffer,
         'http_chunk_size': opts.http_chunk_size,
         'continuedl': opts.continue_dl,
-        'noprogress': opts.noprogress,
+        'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress,
         'progress_with_newline': opts.progress_with_newline,
+        'progress_template': opts.progress_template,
         'playliststart': opts.playliststart,
         'playlistend': opts.playlistend,
         'playlistreverse': opts.playlist_reverse,

yt_dlp/compat.py

@@ -159,6 +159,12 @@ except ImportError:
         compat_pycrypto_AES = None
 
 
+def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
+    if compat_os_name != 'nt':
+        return
+    os.system('')
+
+
 #  Deprecated
 
 compat_basestring = str
@@ -281,5 +287,6 @@ __all__ = [
     'compat_xml_parse_error',
     'compat_xpath',
     'compat_zip',
+    'windows_enable_vt_mode',
     'workaround_optparse_bug9161',
 ]

yt_dlp/downloader/common.py

@@ -7,7 +7,6 @@ import sys
 import time
 import random
 
-from ..compat import compat_os_name
 from ..utils import (
     decodeArgument,
     encodeFilename,
@@ -17,6 +16,7 @@ from ..utils import (
     timeconvert,
 )
 from ..minicurses import (
+    MultilineLogger,
     MultilinePrinter,
     QuietMultilinePrinter,
     BreaklineStatusPrinter
@@ -44,8 +44,6 @@ class FileDownloader(object):
     noresizebuffer:     Do not automatically resize the download buffer.
     continuedl:         Try to continue downloads if possible.
     noprogress:         Do not print the progress bar.
-    logtostderr:        Log messages to stderr instead of stdout.
-    consoletitle:       Display progress in console window's titlebar.
     nopart:             Do not use temporary .part files.
     updatetime:         Use the Last-modified header to set output file timestamps.
     test:               Download only first bytes to test the downloader.
@@ -61,6 +59,7 @@ class FileDownloader(object):
     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
                         useful for bypassing bandwidth throttling imposed by
                         a webserver (experimental)
+    progress_template:  See YoutubeDL.py
 
     Subclasses of this one must re-define the real_download method.
     """
@@ -73,7 +72,7 @@ class FileDownloader(object):
         self.ydl = ydl
         self._progress_hooks = []
         self.params = params
-        self._multiline = None
+        self._prepare_multiline_status()
         self.add_progress_hook(self.report_progress)
 
     @staticmethod
@@ -242,55 +241,46 @@ class FileDownloader(object):
         """Report destination filename."""
         self.to_screen('[download] Destination: ' + filename)
 
-    def _prepare_multiline_status(self, lines):
+    def _prepare_multiline_status(self, lines=1):
-        if self.params.get('quiet'):
+        if self.params.get('noprogress'):
             self._multiline = QuietMultilinePrinter()
-        elif self.params.get('progress_with_newline', False):
+        elif self.ydl.params.get('logger'):
+            self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
+        elif self.params.get('progress_with_newline'):
             self._multiline = BreaklineStatusPrinter(sys.stderr, lines)
-        elif self.params.get('noprogress', False):
-            self._multiline = None
         else:
-            self._multiline = MultilinePrinter(sys.stderr, lines)
+            self._multiline = MultilinePrinter(sys.stderr, lines, not self.params.get('quiet'))
 
     def _finish_multiline_status(self):
-        if self._multiline is not None:
+        self._multiline.end()
-            self._multiline.end()
+
-
+    def _report_progress_status(self, s):
-    def _report_progress_status(self, msg, is_last_line=False, progress_line=None):
+        progress_dict = s.copy()
-        fullmsg = '[download] ' + msg
+        progress_dict.pop('info_dict')
-        if self.params.get('progress_with_newline', False):
+        progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
-            self.to_screen(fullmsg)
+
-        elif progress_line is not None and self._multiline is not None:
+        progress_template = self.params.get('progress_template', {})
-            self._multiline.print_at_line(fullmsg, progress_line)
+        self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
-        else:
+            progress_template.get('download') or '[download] %(progress._default_template)s',
-            if compat_os_name == 'nt' or not sys.stderr.isatty():
+            progress_dict), s.get('progress_idx') or 0)
-                prev_len = getattr(self, '_report_progress_prev_line_length', 0)
+        self.to_console_title(self.ydl.evaluate_outtmpl(
-                if prev_len > len(fullmsg):
+            progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
-                    fullmsg += ' ' * (prev_len - len(fullmsg))
+            progress_dict))
-                self._report_progress_prev_line_length = len(fullmsg)
-                clear_line = '\r'
-            else:
-                clear_line = '\r\x1b[K'
-            self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
-        self.to_console_title('yt-dlp ' + msg)
 
     def report_progress(self, s):
         if s['status'] == 'finished':
-            if self.params.get('noprogress', False):
+            if self.params.get('noprogress'):
                 self.to_screen('[download] Download completed')
-            else:
+            msg_template = '100%%'
-                msg_template = '100%%'
+            if s.get('total_bytes') is not None:
-                if s.get('total_bytes') is not None:
+                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
-                    s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                msg_template += ' of %(_total_bytes_str)s'
-                    msg_template += ' of %(_total_bytes_str)s'
+            if s.get('elapsed') is not None:
-                if s.get('elapsed') is not None:
+                s['_elapsed_str'] = self.format_seconds(s['elapsed'])
-                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                msg_template += ' in %(_elapsed_str)s'
-                    msg_template += ' in %(_elapsed_str)s'
+            s['_percent_str'] = self.format_percent(100)
-                self._report_progress_status(
+            s['_default_template'] = msg_template % s
-                    msg_template % s, is_last_line=True, progress_line=s.get('progress_idx'))
+            self._report_progress_status(s)
-            return
-
-        if self.params.get('noprogress'):
             return
 
         if s['status'] != 'downloading':
@@ -332,8 +322,8 @@ class FileDownloader(object):
                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
             else:
                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
-
+        s['_default_template'] = msg_template % s
-        self._report_progress_status(msg_template % s, progress_line=s.get('progress_idx'))
+        self._report_progress_status(s)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
@@ -405,7 +395,9 @@ class FileDownloader(object):
                     '[download] Sleeping %s seconds ...' % (
                         sleep_interval_sub))
                 time.sleep(sleep_interval_sub)
-        return self.real_download(filename, info_dict), True
+        ret = self.real_download(filename, info_dict)
+        self._finish_multiline_status()
+        return ret, True
 
     def real_download(self, filename, info_dict):
         """Real download process. Redefine in subclasses."""

yt_dlp/downloader/fragment.py

@@ -393,9 +393,7 @@ class FragmentFD(FileDownloader):
                 result = result and job.result()
             finally:
                 tpe.shutdown(wait=True)
-
+        return result
-        self._finish_multiline_status()
-        return True
 
     def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None):
         fragment_retries = self.params.get('fragment_retries', 0)

yt_dlp/extractor/common.py

@@ -1134,10 +1134,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
+        _name = self._downloader._color_text(name, 'blue')
-            _name = '\033[0;34m%s\033[0m' % name
-        else:
-            _name = name
 
         if mobj:
             if group is None:

yt_dlp/minicurses.py

@@ -1,10 +1,12 @@
-import os
-
 from threading import Lock
-from .utils import compat_os_name, get_windows_version
+from .utils import supports_terminal_sequences, TERMINAL_SEQUENCES
+
 
+class MultilinePrinterBase:
+    def __init__(self, stream=None, lines=1):
+        self.stream = stream
+        self.maximum = lines - 1
 
-class MultilinePrinterBase():
     def __enter__(self):
         return self
 
@@ -17,119 +19,87 @@ class MultilinePrinterBase():
     def end(self):
         pass
 
+    def _add_line_number(self, text, line):
+        if self.maximum:
+            return f'{line + 1}: {text}'
+        return text
 
-class MultilinePrinter(MultilinePrinterBase):
 
-    def __init__(self, stream, lines):
+class QuietMultilinePrinter(MultilinePrinterBase):
-        """
+    pass
-        @param stream stream to write to
-        @lines number of lines to be written
-        """
-        self.stream = stream
 
-        is_win10 = compat_os_name == 'nt' and get_windows_version() >= (10, )
-        self.CARRIAGE_RETURN = '\r'
-        if os.getenv('TERM') and self._isatty() or is_win10:
-            # reason not to use curses https://github.com/yt-dlp/yt-dlp/pull/1036#discussion_r713851492
-            # escape sequences for Win10 https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
-            self.UP = '\x1b[A'
-            self.DOWN = '\n'
-            self.ERASE_LINE = '\x1b[K'
-            self._HAVE_FULLCAP = self._isatty() or is_win10
-        else:
-            self.UP = self.DOWN = self.ERASE_LINE = None
-            self._HAVE_FULLCAP = False
 
-        # lines are numbered from top to bottom, counting from 0 to self.maximum
+class MultilineLogger(MultilinePrinterBase):
-        self.maximum = lines - 1
+    def print_at_line(self, text, pos):
-        self.lastline = 0
+        # stream is the logger object, not an actual stream
-        self.lastlength = 0
+        self.stream.debug(self._add_line_number(text, pos))
 
-        self.movelock = Lock()
 
-    @property
+class BreaklineStatusPrinter(MultilinePrinterBase):
-    def have_fullcap(self):
+    def print_at_line(self, text, pos):
-        """
+        self.stream.write(self._add_line_number(text, pos) + '\n')
-        True if the TTY is allowing to control cursor,
-        so that multiline progress works
-        """
-        return self._HAVE_FULLCAP
 
-    def _isatty(self):
+
-        try:
+class MultilinePrinter(MultilinePrinterBase):
-            return self.stream.isatty()
+    def __init__(self, stream=None, lines=1, preserve_output=True):
-        except BaseException:
+        super().__init__(stream, lines)
-            return False
+        self.preserve_output = preserve_output
+        self._lastline = self._lastlength = 0
+        self._movelock = Lock()
+        self._HAVE_FULLCAP = supports_terminal_sequences(self.stream)
+
+    def lock(func):
+        def wrapper(self, *args, **kwargs):
+            with self._movelock:
+                return func(self, *args, **kwargs)
+        return wrapper
 
     def _move_cursor(self, dest):
-        current = min(self.lastline, self.maximum)
+        current = min(self._lastline, self.maximum)
-        self.stream.write(self.CARRIAGE_RETURN)
+        self.stream.write('\r')
-        if current == dest:
+        distance = dest - current
-            # current and dest are at same position, no need to move cursor
+        if distance < 0:
+            self.stream.write(TERMINAL_SEQUENCES['UP'] * -distance)
+        elif distance > 0:
+            self.stream.write(TERMINAL_SEQUENCES['DOWN'] * distance)
+        self._lastline = dest
+
+    @lock
+    def print_at_line(self, text, pos):
+        if self._HAVE_FULLCAP:
+            self._move_cursor(pos)
+            self.stream.write(TERMINAL_SEQUENCES['ERASE_LINE'])
+            self.stream.write(text)
             return
-        elif current > dest:
-            # when maximum == 2,
-            # 0. dest
-            # 1.
-            # 2. current
-            self.stream.write(self.UP * (current - dest))
-        elif current < dest:
-            # when maximum == 2,
-            # 0. current
-            # 1.
-            # 2. dest
-            self.stream.write(self.DOWN * (dest - current))
-        self.lastline = dest
 
-    def print_at_line(self, text, pos):
+        text = self._add_line_number(text, pos)
-        with self.movelock:
+        textlen = len(text)
-            if self.have_fullcap:
+        if self._lastline == pos:
-                self._move_cursor(pos)
+            # move cursor at the start of progress when writing to same line
-                self.stream.write(self.ERASE_LINE)
+            self.stream.write('\r')
-                self.stream.write(text)
+            if self._lastlength > textlen:
-            else:
+                text += ' ' * (self._lastlength - textlen)
-                if self.maximum != 0:
+            self._lastlength = textlen
-                    # let user know about which line is updating the status
+        else:
-                    text = f'{pos + 1}: {text}'
+            # otherwise, break the line
-                textlen = len(text)
+            self.stream.write('\n')
-                if self.lastline == pos:
+            self._lastlength = textlen
-                    # move cursor at the start of progress when writing to same line
+        self.stream.write(text)
-                    self.stream.write(self.CARRIAGE_RETURN)
+        self._lastline = pos
-                    if self.lastlength > textlen:
-                        text += ' ' * (self.lastlength - textlen)
-                    self.lastlength = textlen
-                else:
-                    # otherwise, break the line
-                    self.stream.write('\n')
-                    self.lastlength = 0
-                self.stream.write(text)
-                self.lastline = pos
 
+    @lock
     def end(self):
-        with self.movelock:
+        # move cursor to the end of the last line, and write line break
-            # move cursor to the end of the last line, and write line break
+        # so that other to_screen calls can precede
-            # so that other to_screen calls can precede
+        if self._HAVE_FULLCAP:
             self._move_cursor(self.maximum)
+        if self.preserve_output:
             self.stream.write('\n')
+            return
 
-
+        if self._HAVE_FULLCAP:
-class QuietMultilinePrinter(MultilinePrinterBase):
+            self.stream.write(
-    def __init__(self):
+                TERMINAL_SEQUENCES['ERASE_LINE']
-        self.have_fullcap = True
+                + f'{TERMINAL_SEQUENCES["UP"]}{TERMINAL_SEQUENCES["ERASE_LINE"]}' * self.maximum)
-
+        else:
-
+            self.stream.write(' ' * self._lastlength)
-class BreaklineStatusPrinter(MultilinePrinterBase):
-
-    def __init__(self, stream, lines):
-        """
-        @param stream stream to write to
-        """
-        self.stream = stream
-        self.maximum = lines
-        self.have_fullcap = True
-
-    def print_at_line(self, text, pos):
-        if self.maximum != 0:
-            # let user know about which line is updating the status
-            text = f'{pos + 1}: {text}'
-        self.stream.write(text + '\n')

yt_dlp/options.py

@@ -910,12 +910,30 @@ def parseOpts(overrideArguments=None):
         help='Output progress bar as new lines')
     verbosity.add_option(
         '--no-progress',
-        action='store_true', dest='noprogress', default=False,
+        action='store_true', dest='noprogress', default=None,
         help='Do not print progress bar')
+    verbosity.add_option(
+        '--progress',
+        action='store_false', dest='noprogress',
+        help='Show progress bar, even if in quiet mode')
     verbosity.add_option(
         '--console-title',
         action='store_true', dest='consoletitle', default=False,
         help='Display progress in console titlebar')
+    verbosity.add_option(
+        '--progress-template',
+        metavar='[TYPES:]TEMPLATE', dest='progress_template', default={}, type='str',
+        action='callback', callback=_dict_from_options_callback,
+        callback_kwargs={
+            'allowed_keys': '(download|postprocess)(-title)?',
+            'default_key': 'download'
+        }, help=(
+            'Template for progress outputs, optionally prefixed with one of "download:" (default), '
+            '"download-title:" (the console title), "postprocess:",  or "postprocess-title:". '
+            'The video\'s fields are accessible under the "info" key and '
+            'the progress attributes are accessible under "progress" key. Eg: '
+            # TODO: Document the fields inside "progress"
+            '--console-title --progress-template "download-title:%(info.id)s-%(progress.eta)s"'))
     verbosity.add_option(
         '-v', '--verbose',
         action='store_true', dest='verbose', default=False,

yt_dlp/postprocessor/common.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import copy
 import functools
 import os
 
@@ -11,7 +12,26 @@ from ..utils import (
 )
 
 
-class PostProcessor(object):
+class PostProcessorMetaClass(type):
+    @staticmethod
+    def run_wrapper(func):
+        @functools.wraps(func)
+        def run(self, info, *args, **kwargs):
+            self._hook_progress({'status': 'started'}, info)
+            ret = func(self, info, *args, **kwargs)
+            if ret is not None:
+                _, info = ret
+            self._hook_progress({'status': 'finished'}, info)
+            return ret
+        return run
+
+    def __new__(cls, name, bases, attrs):
+        if 'run' in attrs:
+            attrs['run'] = cls.run_wrapper(attrs['run'])
+        return type.__new__(cls, name, bases, attrs)
+
+
+class PostProcessor(metaclass=PostProcessorMetaClass):
     """Post Processor class.
 
     PostProcessor objects can be added to downloaders with their
@@ -34,7 +54,9 @@ class PostProcessor(object):
     _downloader = None
 
     def __init__(self, downloader=None):
-        self._downloader = downloader
+        self._progress_hooks = []
+        self.add_progress_hook(self.report_progress)
+        self.set_downloader(downloader)
         self.PP_NAME = self.pp_key()
 
     @classmethod
@@ -68,6 +90,10 @@ class PostProcessor(object):
     def set_downloader(self, downloader):
         """Sets the downloader for this PP."""
         self._downloader = downloader
+        if not downloader:
+            return
+        for ph in downloader._postprocessor_hooks:
+            self.add_progress_hook(ph)
 
     @staticmethod
     def _restrict_to(*, video=True, audio=True, images=True):
@@ -115,6 +141,39 @@ class PostProcessor(object):
         return _configuration_args(
             self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs)
 
+    def _hook_progress(self, status, info_dict):
+        if not self._progress_hooks:
+            return
+        info_dict = dict(info_dict)
+        for key in ('__original_infodict', '__postprocessors'):
+            info_dict.pop(key, None)
+        status.update({
+            'info_dict': copy.deepcopy(info_dict),
+            'postprocessor': self.pp_key(),
+        })
+        for ph in self._progress_hooks:
+            ph(status)
+
+    def add_progress_hook(self, ph):
+        # See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface
+        self._progress_hooks.append(ph)
+
+    def report_progress(self, s):
+        s['_default_template'] = '%(postprocessor)s %(status)s' % s
+
+        progress_dict = s.copy()
+        progress_dict.pop('info_dict')
+        progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
+
+        progress_template = self.get_param('progress_template', {})
+        tmpl = progress_template.get('postprocess')
+        if tmpl:
+            self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict))
+
+        self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
+            progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s',
+            progress_dict))
+
 
 class AudioConversionError(PostProcessingError):
     pass

yt_dlp/postprocessor/metadataparser.py

@@ -62,8 +62,7 @@ class MetadataParserPP(PostProcessor):
 
     def interpretter(self, inp, out):
         def f(info):
-            outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(template, info)
+            data_to_parse = self._downloader.evaluate_outtmpl(template, info)
-            data_to_parse = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
             self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}')
             match = out_re.search(data_to_parse)
             if match is None:

yt_dlp/postprocessor/modify_chapters.py

@@ -292,8 +292,7 @@ class ModifyChaptersPP(FFmpegPostProcessor):
                     'name': SponsorBlockPP.CATEGORIES[category],
                     'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats]
                 })
-                outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(self._sponsorblock_chapter_title, c)
+                c['title'] = self._downloader.evaluate_outtmpl(self._sponsorblock_chapter_title, c)
-                c['title'] = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
                 # Merge identically named sponsors.
                 if (new_chapters and 'categories' in new_chapters[-1]
                         and new_chapters[-1]['title'] == c['title']):

yt_dlp/utils.py

@@ -6440,3 +6440,26 @@ def jwt_encode_hs256(payload_data, key, headers={}):
     signature_b64 = base64.b64encode(h.digest())
     token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64
     return token
+
+
+def supports_terminal_sequences(stream):
+    if compat_os_name == 'nt':
+        if get_windows_version() < (10, ):
+            return False
+    elif not os.getenv('TERM'):
+        return False
+    try:
+        return stream.isatty()
+    except BaseException:
+        return False
+
+
+TERMINAL_SEQUENCES = {
+    'DOWN': '\n',
+    'UP': '\x1b[A',
+    'ERASE_LINE': '\x1b[K',
+    'RED': '\033[0;31m',
+    'YELLOW': '\033[0;33m',
+    'BLUE': '\033[0;34m',
+    'RESET_STYLE': '\033[0m',
+}