Allow users to specify encoding in each config files (#4357)

Authored by: Lesmiscore
pull/4363/head
Lesmiscore 2 years ago committed by GitHub
parent 49afc1d84a
commit a904a7f8c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1161,6 +1161,15 @@ Note that options in configuration file are just the same options aka switches u
You can use `--ignore-config` if you want to disable all configuration files for a particular yt-dlp run. If `--ignore-config` is found inside any configuration file, no further configuration will be loaded. For example, having the option in the portable configuration file prevents loading of home, user, and system configurations. Additionally, (for backward compatibility) if `--ignore-config` is found inside the system configuration file, the user configuration is not loaded.
### Specifying encoding of config files
By default, config files are read in the encoding from system locale.
If you saved your config file in a different encoding than that, you may write `# coding: ENCODING` to the beginning of the file. (e.g. `# coding: shift-jis`)
There must not be any characters before that, including spaces.
If you have BOM enabled, it will be used instead.
### Authentication with `.netrc` file
You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every yt-dlp execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in `--netrc-location` and restrict permissions to read/write by only you:

@ -39,6 +39,7 @@ from yt_dlp.utils import (
datetime_from_str,
detect_exe_version,
determine_ext,
determine_file_encoding,
dfxp2srt,
dict_get,
encode_base_n,
@ -1822,6 +1823,33 @@ Line 1
with contextlib.suppress(OSError):
os.remove(FILE)
def test_determine_file_encoding(self):
self.assertEqual(determine_file_encoding(b''), (None, 0))
self.assertEqual(determine_file_encoding(b'--verbose -x --audio-format mkv\n'), (None, 0))
self.assertEqual(determine_file_encoding(b'\xef\xbb\xbf'), ('utf-8', 3))
self.assertEqual(determine_file_encoding(b'\x00\x00\xfe\xff'), ('utf-32-be', 4))
self.assertEqual(determine_file_encoding(b'\xff\xfe'), ('utf-16-le', 2))
self.assertEqual(determine_file_encoding(b'# -*- coding: cp932 -*-'), ('cp932', 0))
self.assertEqual(determine_file_encoding(b'# -*- coding: cp932 -*-\n'), ('cp932', 0))
self.assertEqual(determine_file_encoding(b'# -*- coding: cp932 -*-\r\n'), ('cp932', 0))
self.assertEqual(determine_file_encoding(b'# coding: utf-8\n--verbose'), ('utf-8', 0))
self.assertEqual(determine_file_encoding(b'# coding: someencodinghere-12345\n--verbose'), ('someencodinghere-12345', 0))
self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932'), ('cp932', 0))
self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932\n'), ('cp932', 0))
self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932\r\n'), ('cp932', 0))
self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932,euc-jp\r\n'), ('cp932', 0))
self.assertEqual(determine_file_encoding(
b'\0\0\0#\0\0\0 \0\0\0c\0\0\0o\0\0\0d\0\0\0i\0\0\0n\0\0\0g\0\0\0:\0\0\0 \0\0\0u\0\0\0t\0\0\0f\0\0\0-\0\0\x003\0\0\x002\0\0\0-\0\0\0b\0\0\0e'),
('utf-32-be', 0))
self.assertEqual(determine_file_encoding(
b'#\0 \0c\0o\0d\0i\0n\0g\0:\0 \0u\0t\0f\0-\x001\x006\0-\0l\0e\0'),
('utf-16-le', 0))
if __name__ == '__main__':
unittest.main()

@ -3485,9 +3485,6 @@ def age_restricted(content_limit, age_limit):
return age_limit < content_limit
def is_html(first_bytes):
""" Detect whether a file contains HTML by examining its first bytes. """
BOMS = [
(b'\xef\xbb\xbf', 'utf-8'),
(b'\x00\x00\xfe\xff', 'utf-32-be'),
@ -3495,6 +3492,11 @@ def is_html(first_bytes):
(b'\xff\xfe', 'utf-16-le'),
(b'\xfe\xff', 'utf-16-be'),
]
""" List of known byte-order-marks (BOM) """
def is_html(first_bytes):
""" Detect whether a file contains HTML by examining its first bytes. """
encoding = 'utf-8'
for bom, enc in BOMS:
@ -5394,6 +5396,41 @@ def read_stdin(what):
return sys.stdin
def determine_file_encoding(data):
"""
From the first 512 bytes of a given file,
it tries to detect the encoding to be used to read as text.
@returns (encoding, bytes to skip)
"""
for bom, enc in BOMS:
# matching BOM beats any declaration
# BOMs are skipped to prevent any errors
if data.startswith(bom):
return enc, len(bom)
# strip off all null bytes to match even when UTF-16 or UTF-32 is used
# endians don't matter
data = data.replace(b'\0', b'')
PREAMBLES = [
# "# -*- coding: utf-8 -*-"
# "# coding: utf-8"
rb'(?m)^#(?:\s+-\*-)?\s*coding\s*:\s*(?P<encoding>\S+)(?:\s+-\*-)?\s*$',
# "# vi: set fileencoding=utf-8"
rb'^#\s+vi\s*:\s+set\s+fileencoding=(?P<encoding>[^\s,]+)'
]
for pb in PREAMBLES:
mobj = re.match(pb, data)
if not mobj:
continue
# preambles aren't skipped since they're just ignored when reading as config
return mobj.group('encoding').decode(), 0
return None, 0
class Config:
own_args = None
parsed_args = None
@ -5445,12 +5482,17 @@ class Config:
@staticmethod
def read_file(filename, default=[]):
try:
optionf = open(filename)
optionf = open(filename, 'rb')
except OSError:
return default # silently skip if file is not present
try:
enc, skip = determine_file_encoding(optionf.read(512))
optionf.seek(skip, io.SEEK_SET)
except OSError:
enc = None # silently skip read errors
try:
# FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
contents = optionf.read()
contents = optionf.read().decode(enc or preferredencoding())
res = shlex.split(contents, comments=True)
except Exception as err:
raise ValueError(f'Unable to parse "{filename}": {err}')

Loading…
Cancel
Save