from __future__ import unicode_literals import os.path import optparse import re import sys from .downloader.external import list_external_downloaders from .compat import ( compat_expanduser, compat_get_terminal_size, compat_getenv, compat_kwargs, compat_shlex_split, ) from .utils import ( preferredencoding, write_string, ) from .version import __version__ def _hide_login_info(opts): PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']) eqre = re.compile('^(?P' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$') def _scrub_eq(o): m = eqre.match(o) if m: return m.group('key') + '=PRIVATE' else: return o opts = list(map(_scrub_eq, opts)) for idx, opt in enumerate(opts): if opt in PRIVATE_OPTS and idx + 1 < len(opts): opts[idx + 1] = 'PRIVATE' return opts def parseOpts(overrideArguments=None): def _readOptions(filename_bytes, default=[]): try: optionf = open(filename_bytes) except IOError: return default # silently skip if file is not present try: # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56 contents = optionf.read() if sys.version_info < (3,): contents = contents.decode(preferredencoding()) res = compat_shlex_split(contents, comments=True) finally: optionf.close() return res def _readUserConf(): xdg_config_home = compat_getenv('XDG_CONFIG_HOME') if xdg_config_home: userConfFile = os.path.join(xdg_config_home, 'youtube-dlc', 'config') if not os.path.isfile(userConfFile): userConfFile = os.path.join(xdg_config_home, 'youtube-dlc.conf') else: userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dlc', 'config') if not os.path.isfile(userConfFile): userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dlc.conf') userConf = _readOptions(userConfFile, None) if userConf is None: appdata_dir = compat_getenv('appdata') if appdata_dir: userConf = _readOptions( os.path.join(appdata_dir, 'youtube-dlc', 'config'), default=None) if userConf is None: userConf = _readOptions( os.path.join(appdata_dir, 'youtube-dlc', 'config.txt'), default=None) if userConf is None: userConf = _readOptions( os.path.join(compat_expanduser('~'), 'youtube-dlc.conf'), default=None) if userConf is None: userConf = _readOptions( os.path.join(compat_expanduser('~'), 'youtube-dlc.conf.txt'), default=None) if userConf is None: userConf = [] return userConf def _format_option_string(option): ''' ('-o', '--option') -> -o, --format METAVAR''' opts = [] if option._short_opts: opts.append(option._short_opts[0]) if option._long_opts: opts.append(option._long_opts[0]) if len(opts) > 1: opts.insert(1, ', ') if option.takes_value(): opts.append(' %s' % option.metavar) return ''.join(opts) def _comma_separated_values_options_callback(option, opt_str, value, parser): setattr(parser.values, option.dest, value.split(',')) # No need to wrap help messages if we're on a wide console columns = compat_get_terminal_size().columns max_width = columns if columns else 80 max_help_position = 80 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) fmt.format_option_strings = _format_option_string kw = { 'version': __version__, 'formatter': fmt, 'usage': '%prog [OPTIONS] URL [URL...]', 'conflict_handler': 'resolve', } parser = optparse.OptionParser(**compat_kwargs(kw)) general = optparse.OptionGroup(parser, 'General Options') general.add_option( '-h', '--help', action='help', help='Print this help text and exit') general.add_option( '--version', action='version', help='Print program version and exit') general.add_option( '-U', '--update', action='store_true', dest='update_self', help='[BROKEN] Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)') general.add_option( '-i', '--ignore-errors', '--no-abort-on-error', action='store_true', dest='ignoreerrors', default=True, help='Continue on download errors, for example to skip unavailable videos in a playlist (default)') general.add_option( '--abort-on-error', '--no-ignore-errors', action='store_false', dest='ignoreerrors', help='Abort downloading of further videos if an error occurs') general.add_option( '--dump-user-agent', action='store_true', dest='dump_user_agent', default=False, help='Display the current browser identification') general.add_option( '--list-extractors', action='store_true', dest='list_extractors', default=False, help='List all supported extractors') general.add_option( '--extractor-descriptions', action='store_true', dest='list_extractor_descriptions', default=False, help='Output descriptions of all supported extractors') general.add_option( '--force-generic-extractor', action='store_true', dest='force_generic_extractor', default=False, help='Force extraction to use the generic extractor') general.add_option( '--default-search', dest='default_search', metavar='PREFIX', help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.') general.add_option( '--ignore-config', '--no-config', action='store_true', help=( 'Do not read configuration files. ' 'When given in the global configuration file /etc/youtube-dl.conf: ' 'Do not read the user configuration in ~/.config/youtube-dl/config ' '(%APPDATA%/youtube-dl/config.txt on Windows)')) general.add_option( '--config-location', dest='config_location', metavar='PATH', help='Location of the configuration file; either the path to the config or its containing directory.') general.add_option( '--flat-playlist', action='store_const', dest='extract_flat', const='in_playlist', default=False, help='Do not extract the videos of a playlist, only list them.') general.add_option( '--flat-videos', action='store_true', dest='extract_flat', help='Do not resolve the video urls') general.add_option( '--no-flat-playlist', action='store_false', dest='extract_flat', help='Extract the videos of a playlist') general.add_option( '--mark-watched', action='store_true', dest='mark_watched', default=False, help='Mark videos watched (YouTube only)') general.add_option( '--no-mark-watched', action='store_false', dest='mark_watched', default=False, help='Do not mark videos watched') general.add_option( '--no-color', '--no-colors', action='store_true', dest='no_color', default=False, help='Do not emit color codes in output') network = optparse.OptionGroup(parser, 'Network Options') network.add_option( '--proxy', dest='proxy', default=None, metavar='URL', help=( 'Use the specified HTTP/HTTPS/SOCKS proxy. To enable ' 'SOCKS proxy, specify a proper scheme. For example ' 'socks5://127.0.0.1:1080/. Pass in an empty string (--proxy "") ' 'for direct connection')) network.add_option( '--socket-timeout', dest='socket_timeout', type=float, default=None, metavar='SECONDS', help='Time to wait before giving up, in seconds') network.add_option( '--source-address', metavar='IP', dest='source_address', default=None, help='Client-side IP address to bind to', ) network.add_option( '-4', '--force-ipv4', action='store_const', const='0.0.0.0', dest='source_address', help='Make all connections via IPv4', ) network.add_option( '-6', '--force-ipv6', action='store_const', const='::', dest='source_address', help='Make all connections via IPv6', ) geo = optparse.OptionGroup(parser, 'Geo Restriction') geo.add_option( '--geo-verification-proxy', dest='geo_verification_proxy', default=None, metavar='URL', help=( 'Use this proxy to verify the IP address for some geo-restricted sites. ' 'The default proxy specified by --proxy (or none, if the option is not present) is used for the actual downloading.')) geo.add_option( '--cn-verification-proxy', dest='cn_verification_proxy', default=None, metavar='URL', help=optparse.SUPPRESS_HELP) geo.add_option( '--geo-bypass', action='store_true', dest='geo_bypass', default=True, help='Bypass geographic restriction via faking X-Forwarded-For HTTP header') geo.add_option( '--no-geo-bypass', action='store_false', dest='geo_bypass', default=True, help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header') geo.add_option( '--geo-bypass-country', metavar='CODE', dest='geo_bypass_country', default=None, help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code') geo.add_option( '--geo-bypass-ip-block', metavar='IP_BLOCK', dest='geo_bypass_ip_block', default=None, help='Force bypass geographic restriction with explicitly provided IP block in CIDR notation') selection = optparse.OptionGroup(parser, 'Video Selection') selection.add_option( '--playlist-start', dest='playliststart', metavar='NUMBER', default=1, type=int, help='Playlist video to start at (default is %default)') selection.add_option( '--playlist-end', dest='playlistend', metavar='NUMBER', default=None, type=int, help='Playlist video to end at (default is last)') selection.add_option( '--playlist-items', dest='playlist_items', metavar='ITEM_SPEC', default=None, help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.') selection.add_option( '--match-title', dest='matchtitle', metavar='REGEX', help='Download only matching titles (regex or caseless sub-string)') selection.add_option( '--reject-title', dest='rejecttitle', metavar='REGEX', help='Skip download for matching titles (regex or caseless sub-string)') selection.add_option( '--max-downloads', dest='max_downloads', metavar='NUMBER', type=int, default=None, help='Abort after downloading NUMBER files') selection.add_option( '--min-filesize', metavar='SIZE', dest='min_filesize', default=None, help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)') selection.add_option( '--max-filesize', metavar='SIZE', dest='max_filesize', default=None, help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)') selection.add_option( '--date', metavar='DATE', dest='date', default=None, help=( 'Download only videos uploaded in this date.' 'The date can be "YYYYMMDD" or in the format' '"(now|today)[+-][0-9](day|week|month|year)(s)?"')) selection.add_option( '--datebefore', metavar='DATE', dest='datebefore', default=None, help=( 'Download only videos uploaded on or before this date. ' 'The date formats accepted is the same as --date')) selection.add_option( '--dateafter', metavar='DATE', dest='dateafter', default=None, help=( 'Download only videos uploaded on or after this date. ' 'The date formats accepted is the same as --date')) selection.add_option( '--min-views', metavar='COUNT', dest='min_views', default=None, type=int, help='Do not download any videos with less than COUNT views') selection.add_option( '--max-views', metavar='COUNT', dest='max_views', default=None, type=int, help='Do not download any videos with more than COUNT views') selection.add_option( '--match-filter', metavar='FILTER', dest='match_filter', default=None, help=( 'Generic video filter. ' 'Specify any key (see the "OUTPUT TEMPLATE" for a list of available keys) to ' 'match if the key is present, ' '!key to check if the key is not present, ' 'key > NUMBER (like "comment_count > 12", also works with ' '>=, <, <=, !=, =) to compare against a number, ' 'key = \'LITERAL\' (like "uploader = \'Mike Smith\'", also works with !=) ' 'to match against a string literal ' 'and & to require multiple matches. ' 'Values which are not known are excluded unless you ' 'put a question mark (?) after the operator. ' 'For example, to only match videos that have been liked more than ' '100 times and disliked less than 50 times (or the dislike ' 'functionality is not available at the given service), but who ' 'also have a description, use --match-filter ' '"like_count > 100 & dislike_count .+?) - (?P.+)"')) postproc.add_option( '--xattrs', action='store_true', dest='xattrs', default=False, help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)') postproc.add_option( '--fixup', metavar='POLICY', dest='fixup', default='detect_or_warn', help=( 'Automatically correct known faults of the file. ' 'One of never (do nothing), warn (only emit a warning), ' 'detect_or_warn (the default; fix file if we can, warn otherwise)')) postproc.add_option( '--prefer-avconv', '--no-prefer-ffmpeg', action='store_false', dest='prefer_ffmpeg', help='Prefer avconv over ffmpeg for running the postprocessors') postproc.add_option( '--prefer-ffmpeg', '--no-prefer-avconv', action='store_true', dest='prefer_ffmpeg', help='Prefer ffmpeg over avconv for running the postprocessors (default)') postproc.add_option( '--ffmpeg-location', '--avconv-location', metavar='PATH', dest='ffmpeg_location', help='Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.') postproc.add_option( '--exec', metavar='CMD', dest='exec_cmd', help='Execute a command on the file after downloading and post-processing, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'') postproc.add_option( '--convert-subs', '--convert-subtitles', metavar='FORMAT', dest='convertsubtitles', default=None, help='Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc)') sponskrub = optparse.OptionGroup(parser, 'SponSkrub Options (SponsorBlock)') sponskrub.add_option( '--sponskrub', action='store_true', dest='sponskrub', default=None, help=( 'Use sponskrub to mark sponsored sections with the data available in SponsorBlock API. ' 'This is enabled by default if the sponskrub binary exists (Youtube only)')) sponskrub.add_option( '--no-sponskrub', action='store_false', dest='sponskrub', help='Do not use sponskrub') sponskrub.add_option( '--sponskrub-cut', default=False, action='store_true', dest='sponskrub_cut', help='Cut out the sponsor sections instead of simply marking them') sponskrub.add_option( '--no-sponskrub-cut', action='store_false', dest='sponskrub_cut', help='Simply mark the sponsor sections, not cut them out (default)') sponskrub.add_option( '--sponskrub-force', default=False, action='store_true', dest='sponskrub_force', help='Run sponskrub even if the video was already downloaded') sponskrub.add_option( '--no-sponskrub-force', action='store_true', dest='sponskrub_force', help='Do not cut out the sponsor sections if the video was already downloaded (default)') sponskrub.add_option( '--sponskrub-location', metavar='PATH', dest='sponskrub_path', default='', help='Location of the sponskrub binary; either the path to the binary or its containing directory.') sponskrub.add_option( '--sponskrub-args', dest='sponskrub_args', metavar='ARGS', help='Give these arguments to sponskrub') extractor = optparse.OptionGroup(parser, 'Extractor Options') extractor.add_option( '--allow-dynamic-mpd', '--no-ignore-dynamic-mpd', action='store_true', dest='dynamic_mpd', default=True, help='Process dynamic DASH manifests (default)') extractor.add_option( '--ignore-dynamic-mpd', '--no-allow-dynamic-mpd', action='store_false', dest='dynamic_mpd', help='Do not process dynamic DASH manifests') parser.add_option_group(general) parser.add_option_group(network) parser.add_option_group(geo) parser.add_option_group(selection) parser.add_option_group(downloader) parser.add_option_group(filesystem) parser.add_option_group(thumbnail) parser.add_option_group(link) parser.add_option_group(verbosity) parser.add_option_group(workarounds) parser.add_option_group(video_format) parser.add_option_group(subtitles) parser.add_option_group(authentication) parser.add_option_group(adobe_pass) parser.add_option_group(postproc) parser.add_option_group(sponskrub) parser.add_option_group(extractor) if overrideArguments is not None: opts, args = parser.parse_args(overrideArguments) if opts.verbose: write_string('[debug] Override config: ' + repr(overrideArguments) + '\n') else: def compat_conf(conf): if sys.version_info < (3,): return [a.decode(preferredencoding(), 'replace') for a in conf] return conf command_line_conf = compat_conf(sys.argv[1:]) opts, args = parser.parse_args(command_line_conf) system_conf = user_conf = custom_conf = [] if '--config-location' in command_line_conf: location = compat_expanduser(opts.config_location) if os.path.isdir(location): location = os.path.join(location, 'youtube-dlc.conf') if not os.path.exists(location): parser.error('config-location %s does not exist.' % location) custom_conf = _readOptions(location) elif '--ignore-config' in command_line_conf: pass else: system_conf = _readOptions('/etc/youtube-dlc.conf') if '--ignore-config' not in system_conf: user_conf = _readUserConf() argv = system_conf + user_conf + custom_conf + command_line_conf opts, args = parser.parse_args(argv) if opts.verbose: for conf_label, conf in ( ('System config', system_conf), ('User config', user_conf), ('Custom config', custom_conf), ('Command-line args', command_line_conf)): write_string('[debug] %s: %s\n' % (conf_label, repr(_hide_login_info(conf)))) return parser, opts, args