mirror of https://github.com/yt-dlp/yt-dlp
[ie/youtube] Implement external n/sig solver (#14157)
Closes #14404, Closes #14431, Closes #14680, Closes #14707 Authored by: bashonly, coletdjnz, seproDev, Grub4K Co-authored-by: coletdjnz <coletdjnz@protonmail.com> Co-authored-by: bashonly <bashonly@protonmail.com> Co-authored-by: sepro <sepro@sepr0.com>pull/8995/merge
Simon Sawicki
1 month ago
committed by
GitHub
parent
d6ee677253
commit
6224a38988
@ -0,0 +1,77 @@
|
||||
name: Challenge Tests
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- .github/workflows/challenge-tests.yml
|
||||
- test/test_jsc/*.py
|
||||
- yt_dlp/extractor/youtube/jsc/**.js
|
||||
- yt_dlp/extractor/youtube/jsc/**.py
|
||||
- yt_dlp/extractor/youtube/pot/**.py
|
||||
- yt_dlp/utils/_jsruntime.py
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/challenge-tests.yml
|
||||
- test/test_jsc/*.py
|
||||
- yt_dlp/extractor/youtube/jsc/**.js
|
||||
- yt_dlp/extractor/youtube/jsc/**.py
|
||||
- yt_dlp/extractor/youtube/pot/**.py
|
||||
- yt_dlp/utils/_jsruntime.py
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: challenge-tests-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Challenge Tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', pypy-3.11]
|
||||
env:
|
||||
QJS_VERSION: '2025-04-26' # Earliest version with rope strings
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install Deno
|
||||
uses: denoland/setup-deno@v2
|
||||
with:
|
||||
deno-version: '2.0.0' # minimum supported version
|
||||
- name: Install Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
# minimum supported version is 1.0.31 but earliest available Windows version is 1.1.0
|
||||
bun-version: ${{ (matrix.os == 'windows-latest' && '1.1.0') || '1.0.31' }}
|
||||
- name: Install Node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20.0' # minimum supported version
|
||||
- name: Install QuickJS (Linux)
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
run: |
|
||||
wget "https://bellard.org/quickjs/binary_releases/quickjs-linux-x86_64-${QJS_VERSION}.zip" -O quickjs.zip
|
||||
unzip quickjs.zip qjs
|
||||
sudo install qjs /usr/local/bin/qjs
|
||||
- name: Install QuickJS (Windows)
|
||||
if: matrix.os == 'windows-latest'
|
||||
shell: pwsh
|
||||
run: |
|
||||
Invoke-WebRequest "https://bellard.org/quickjs/binary_releases/quickjs-win-x86_64-${Env:QJS_VERSION}.zip" -OutFile quickjs.zip
|
||||
unzip quickjs.zip
|
||||
- name: Install test requirements
|
||||
run: |
|
||||
python ./devscripts/install_deps.py --print --only-optional-groups --include-group test > requirements.txt
|
||||
python ./devscripts/install_deps.py --print -c certifi -c requests -c urllib3 -c yt-dlp-ejs >> requirements.txt
|
||||
python -m pip install -U -r requirements.txt
|
||||
- name: Run tests
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
python -m yt_dlp -v --js-runtimes node --js-runtimes bun --js-runtimes quickjs || true
|
||||
python ./devscripts/run_tests.py test/test_jsc -k download
|
||||
@ -1,41 +0,0 @@
|
||||
name: Signature Tests
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- .github/workflows/signature-tests.yml
|
||||
- test/test_youtube_signature.py
|
||||
- yt_dlp/jsinterp.py
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/signature-tests.yml
|
||||
- test/test_youtube_signature.py
|
||||
- yt_dlp/jsinterp.py
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: signature-tests-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Signature Tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', pypy-3.11]
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install test requirements
|
||||
run: python ./devscripts/install_deps.py --only-optional-groups --include-group test
|
||||
- name: Run tests
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
python3 -m yt_dlp -v || true # Print debug head
|
||||
python3 ./devscripts/run_tests.py test/test_youtube_signature.py
|
||||
@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import hashlib
|
||||
import pathlib
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
|
||||
TEMPLATE = '''\
|
||||
# This file is generated by devscripts/update_ejs.py. DO NOT MODIFY!
|
||||
|
||||
VERSION = {version!r}
|
||||
HASHES = {{
|
||||
{hash_mapping}
|
||||
}}
|
||||
'''
|
||||
PREFIX = ' "yt-dlp-ejs=='
|
||||
BASE_PATH = pathlib.Path(__file__).parent.parent
|
||||
PYPROJECT_PATH = BASE_PATH / 'pyproject.toml'
|
||||
PACKAGE_PATH = BASE_PATH / 'yt_dlp/extractor/youtube/jsc/_builtin/vendor'
|
||||
RELEASE_URL = 'https://api.github.com/repos/yt-dlp/ejs/releases/latest'
|
||||
ASSETS = {
|
||||
'yt.solver.lib.js': False,
|
||||
'yt.solver.lib.min.js': False,
|
||||
'yt.solver.deno.lib.js': True,
|
||||
'yt.solver.bun.lib.js': True,
|
||||
'yt.solver.core.min.js': False,
|
||||
'yt.solver.core.js': True,
|
||||
}
|
||||
MAKEFILE_PATH = BASE_PATH / 'Makefile'
|
||||
|
||||
|
||||
def request(url: str):
|
||||
return contextlib.closing(urllib.request.urlopen(url))
|
||||
|
||||
|
||||
def makefile_variables(
|
||||
version: str | None = None,
|
||||
name: str | None = None,
|
||||
digest: str | None = None,
|
||||
data: bytes | None = None,
|
||||
keys_only: bool = False,
|
||||
) -> dict[str, str | None]:
|
||||
assert keys_only or all(arg is not None for arg in (version, name, digest, data))
|
||||
|
||||
return {
|
||||
'EJS_VERSION': None if keys_only else version,
|
||||
'EJS_WHEEL_NAME': None if keys_only else name,
|
||||
'EJS_WHEEL_HASH': None if keys_only else digest,
|
||||
'EJS_PY_FOLDERS': None if keys_only else list_wheel_contents(data, 'py', files=False),
|
||||
'EJS_PY_FILES': None if keys_only else list_wheel_contents(data, 'py', folders=False),
|
||||
'EJS_JS_FOLDERS': None if keys_only else list_wheel_contents(data, 'js', files=False),
|
||||
'EJS_JS_FILES': None if keys_only else list_wheel_contents(data, 'js', folders=False),
|
||||
}
|
||||
|
||||
|
||||
def list_wheel_contents(
|
||||
wheel_data: bytes,
|
||||
suffix: str | None = None,
|
||||
folders: bool = True,
|
||||
files: bool = True,
|
||||
) -> str:
|
||||
assert folders or files, 'at least one of "folders" or "files" must be True'
|
||||
|
||||
path_gen = (zinfo.filename for zinfo in zipfile.ZipFile(io.BytesIO(wheel_data)).infolist())
|
||||
filtered = filter(lambda path: path.startswith('yt_dlp_ejs/'), path_gen)
|
||||
if suffix:
|
||||
filtered = filter(lambda path: path.endswith(f'.{suffix}'), filtered)
|
||||
|
||||
files_list = list(filtered)
|
||||
if not folders:
|
||||
return ' '.join(files_list)
|
||||
|
||||
folders_list = list(dict.fromkeys(path.rpartition('/')[0] for path in files_list))
|
||||
if not files:
|
||||
return ' '.join(folders_list)
|
||||
|
||||
return ' '.join(folders_list + files_list)
|
||||
|
||||
|
||||
def main():
|
||||
current_version = None
|
||||
with PYPROJECT_PATH.open() as file:
|
||||
for line in file:
|
||||
if not line.startswith(PREFIX):
|
||||
continue
|
||||
current_version, _, _ = line.removeprefix(PREFIX).partition('"')
|
||||
|
||||
if not current_version:
|
||||
print('yt-dlp-ejs dependency line could not be found')
|
||||
return
|
||||
|
||||
makefile_info = makefile_variables(keys_only=True)
|
||||
prefixes = tuple(f'{key} = ' for key in makefile_info)
|
||||
with MAKEFILE_PATH.open() as file:
|
||||
for line in file:
|
||||
if not line.startswith(prefixes):
|
||||
continue
|
||||
key, _, val = line.partition(' = ')
|
||||
makefile_info[key] = val.rstrip()
|
||||
|
||||
with request(RELEASE_URL) as resp:
|
||||
info = json.load(resp)
|
||||
|
||||
version = info['tag_name']
|
||||
if version == current_version:
|
||||
print(f'yt-dlp-ejs is up to date! ({version})')
|
||||
return
|
||||
|
||||
print(f'Updating yt-dlp-ejs from {current_version} to {version}')
|
||||
hashes = []
|
||||
wheel_info = {}
|
||||
for asset in info['assets']:
|
||||
name = asset['name']
|
||||
is_wheel = name.startswith('yt_dlp_ejs-') and name.endswith('.whl')
|
||||
if not is_wheel and name not in ASSETS:
|
||||
continue
|
||||
with request(asset['browser_download_url']) as resp:
|
||||
data = resp.read()
|
||||
|
||||
# verify digest from github
|
||||
digest = asset['digest']
|
||||
algo, _, expected = digest.partition(':')
|
||||
hexdigest = hashlib.new(algo, data).hexdigest()
|
||||
assert hexdigest == expected, f'downloaded attest mismatch ({hexdigest!r} != {expected!r})'
|
||||
|
||||
if is_wheel:
|
||||
wheel_info = makefile_variables(version, name, digest, data)
|
||||
continue
|
||||
|
||||
# calculate sha3-512 digest
|
||||
asset_hash = hashlib.sha3_512(data).hexdigest()
|
||||
hashes.append(f' {name!r}: {asset_hash!r},')
|
||||
|
||||
if ASSETS[name]:
|
||||
(PACKAGE_PATH / name).write_bytes(data)
|
||||
|
||||
hash_mapping = '\n'.join(hashes)
|
||||
for asset_name in ASSETS:
|
||||
assert asset_name in hash_mapping, f'{asset_name} not found in release'
|
||||
|
||||
assert all(wheel_info.get(key) for key in makefile_info), 'wheel info not found in release'
|
||||
|
||||
(PACKAGE_PATH / '_info.py').write_text(TEMPLATE.format(
|
||||
version=version,
|
||||
hash_mapping=hash_mapping,
|
||||
))
|
||||
|
||||
content = PYPROJECT_PATH.read_text()
|
||||
updated = content.replace(PREFIX + current_version, PREFIX + version)
|
||||
PYPROJECT_PATH.write_text(updated)
|
||||
|
||||
makefile = MAKEFILE_PATH.read_text()
|
||||
for key in wheel_info:
|
||||
makefile = makefile.replace(f'{key} = {makefile_info[key]}', f'{key} = {wheel_info[key]}')
|
||||
MAKEFILE_PATH.write_text(makefile)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,60 @@
|
||||
import re
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
import yt_dlp.globals
|
||||
from yt_dlp import YoutubeDL
|
||||
from yt_dlp.extractor.common import InfoExtractor
|
||||
|
||||
|
||||
_TESTDATA_PATH = pathlib.Path(__file__).parent.parent / 'testdata/sigs'
|
||||
_player_re = re.compile(r'^.+/player/(?P<id>[a-zA-Z0-9_/.-]+)\.js$')
|
||||
_player_id_trans = str.maketrans(dict.fromkeys('/.-', '_'))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ie() -> InfoExtractor:
|
||||
runtime_names = yt_dlp.globals.supported_js_runtimes.value
|
||||
ydl = YoutubeDL({'js_runtimes': {key: {} for key in runtime_names}})
|
||||
ie = ydl.get_info_extractor('Youtube')
|
||||
|
||||
def _load_player(video_id, player_url, fatal=True):
|
||||
match = _player_re.match(player_url)
|
||||
test_id = match.group('id').translate(_player_id_trans)
|
||||
cached_file = _TESTDATA_PATH / f'player-{test_id}.js'
|
||||
|
||||
if cached_file.exists():
|
||||
return cached_file.read_text()
|
||||
|
||||
if code := ie._download_webpage(player_url, video_id, fatal=fatal):
|
||||
_TESTDATA_PATH.mkdir(exist_ok=True, parents=True)
|
||||
cached_file.write_text(code)
|
||||
return code
|
||||
|
||||
return None
|
||||
|
||||
ie._load_player = _load_player
|
||||
return ie
|
||||
|
||||
|
||||
class MockLogger:
|
||||
def trace(self, message: str):
|
||||
print(f'trace: {message}')
|
||||
|
||||
def debug(self, message: str, *, once=False):
|
||||
print(f'debug: {message}')
|
||||
|
||||
def info(self, message: str):
|
||||
print(f'info: {message}')
|
||||
|
||||
def warning(self, message: str, *, once=False):
|
||||
print(f'warning: {message}')
|
||||
|
||||
def error(self, message: str):
|
||||
print(f'error: {message}')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def logger():
|
||||
return MockLogger()
|
||||
@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import enum
|
||||
import importlib.util
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeRequest,
|
||||
JsChallengeType,
|
||||
JsChallengeProviderResponse,
|
||||
JsChallengeResponse,
|
||||
NChallengeInput,
|
||||
NChallengeOutput,
|
||||
SigChallengeInput,
|
||||
SigChallengeOutput,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.bun import BunJCP
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.deno import DenoJCP
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.node import NodeJCP
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.quickjs import QuickJSJCP
|
||||
|
||||
|
||||
_has_ejs = bool(importlib.util.find_spec('yt_dlp_ejs'))
|
||||
pytestmark = pytest.mark.skipif(not _has_ejs, reason='yt-dlp-ejs not available')
|
||||
|
||||
|
||||
class Variant(enum.Enum):
|
||||
main = 'player_ias.vflset/en_US/base.js'
|
||||
tcc = 'player_ias_tcc.vflset/en_US/base.js'
|
||||
tce = 'player_ias_tce.vflset/en_US/base.js'
|
||||
es5 = 'player_es5.vflset/en_US/base.js'
|
||||
es6 = 'player_es6.vflset/en_US/base.js'
|
||||
tv = 'tv-player-ias.vflset/tv-player-ias.js'
|
||||
tv_es6 = 'tv-player-es6.vflset/tv-player-es6.js'
|
||||
phone = 'player-plasma-ias-phone-en_US.vflset/base.js'
|
||||
tablet = 'player-plasma-ias-tablet-en_US.vflset/base.js'
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Challenge:
|
||||
player: str
|
||||
variant: Variant
|
||||
type: JsChallengeType
|
||||
values: dict[str, str] = dataclasses.field(default_factory=dict)
|
||||
|
||||
def url(self, /):
|
||||
return f'https://www.youtube.com/s/player/{self.player}/{self.variant.value}'
|
||||
|
||||
|
||||
CHALLENGES: list[Challenge] = [
|
||||
Challenge('3d3ba064', Variant.tce, JsChallengeType.N, {
|
||||
'ZdZIqFPQK-Ty8wId': 'qmtUsIz04xxiNW',
|
||||
'4GMrWHyKI5cEvhDO': 'N9gmEX7YhKTSmw',
|
||||
}),
|
||||
Challenge('3d3ba064', Variant.tce, JsChallengeType.SIG, {
|
||||
'gN7a-hudCuAuPH6fByOk1_GNXN0yNMHShjZXS2VOgsEItAJz0tipeavEOmNdYN-wUtcEqD3bCXjc0iyKfAyZxCBGgIARwsSdQfJ2CJtt':
|
||||
'ttJC2JfQdSswRAIgGBCxZyAfKyi0cjXCb3gqEctUw-NYdNmOEvaepit0zJAtIEsgOV2SXZjhSHMNy0NXNG_1kNyBf6HPuAuCduh-a7O',
|
||||
}),
|
||||
Challenge('5ec65609', Variant.tce, JsChallengeType.N, {
|
||||
'0eRGgQWJGfT5rFHFj': '4SvMpDQH-vBJCw',
|
||||
}),
|
||||
Challenge('5ec65609', Variant.tce, JsChallengeType.SIG, {
|
||||
'AAJAJfQdSswRQIhAMG5SN7-cAFChdrE7tLA6grH0rTMICA1mmDc0HoXgW3CAiAQQ4=CspfaF_vt82XH5yewvqcuEkvzeTsbRuHssRMyJQ=I':
|
||||
'AJfQdSswRQIhAMG5SN7-cAFChdrE7tLA6grI0rTMICA1mmDc0HoXgW3CAiAQQ4HCspfaF_vt82XH5yewvqcuEkvzeTsbRuHssRMyJQ==',
|
||||
}),
|
||||
Challenge('6742b2b9', Variant.tce, JsChallengeType.N, {
|
||||
'_HPB-7GFg1VTkn9u': 'qUAsPryAO_ByYg',
|
||||
'K1t_fcB6phzuq2SF': 'Y7PcOt3VE62mog',
|
||||
}),
|
||||
Challenge('6742b2b9', Variant.tce, JsChallengeType.SIG, {
|
||||
'MMGZJMUucirzS_SnrSPYsc85CJNnTUi6GgR5NKn-znQEICACojE8MHS6S7uYq4TGjQX_D4aPk99hNU6wbTvorvVVMgIARwsSdQfJAA':
|
||||
'AJfQdSswRAIgMVVvrovTbw6UNh99kPa4D_XQjGT4qYu7S6SHM8EjoCACIEQnz-nKN5RgG6iUTnNJC58csYPSrnS_SzricuUMJZGM',
|
||||
}),
|
||||
Challenge('2b83d2e0', Variant.main, JsChallengeType.N, {
|
||||
'0eRGgQWJGfT5rFHFj': 'euHbygrCMLksxd',
|
||||
}),
|
||||
Challenge('2b83d2e0', Variant.main, JsChallengeType.SIG, {
|
||||
'MMGZJMUucirzS_SnrSPYsc85CJNnTUi6GgR5NKn-znQEICACojE8MHS6S7uYq4TGjQX_D4aPk99hNU6wbTvorvVVMgIARwsSdQfJA':
|
||||
'-MGZJMUucirzS_SnrSPYsc85CJNnTUi6GgR5NKnMznQEICACojE8MHS6S7uYq4TGjQX_D4aPk99hNU6wbTvorvVVMgIARwsSdQfJ',
|
||||
}),
|
||||
Challenge('638ec5c6', Variant.main, JsChallengeType.N, {
|
||||
'ZdZIqFPQK-Ty8wId': '1qov8-KM-yH',
|
||||
}),
|
||||
Challenge('638ec5c6', Variant.main, JsChallengeType.SIG, {
|
||||
'gN7a-hudCuAuPH6fByOk1_GNXN0yNMHShjZXS2VOgsEItAJz0tipeavEOmNdYN-wUtcEqD3bCXjc0iyKfAyZxCBGgIARwsSdQfJ2CJtt':
|
||||
'MhudCuAuP-6fByOk1_GNXN7gNHHShjyXS2VOgsEItAJz0tipeav0OmNdYN-wUtcEqD3bCXjc0iyKfAyZxCBGgIARwsSdQfJ2CJtt',
|
||||
}),
|
||||
]
|
||||
|
||||
requests: list[JsChallengeRequest] = []
|
||||
responses: list[JsChallengeProviderResponse] = []
|
||||
for test in CHALLENGES:
|
||||
input_type, output_type = {
|
||||
JsChallengeType.N: (NChallengeInput, NChallengeOutput),
|
||||
JsChallengeType.SIG: (SigChallengeInput, SigChallengeOutput),
|
||||
}[test.type]
|
||||
|
||||
request = JsChallengeRequest(test.type, input_type(test.url(), list(test.values.keys())), test.player)
|
||||
requests.append(request)
|
||||
responses.append(JsChallengeProviderResponse(request, JsChallengeResponse(test.type, output_type(test.values))))
|
||||
|
||||
|
||||
@pytest.fixture(params=[BunJCP, DenoJCP, NodeJCP, QuickJSJCP])
|
||||
def jcp(request, ie, logger):
|
||||
obj = request.param(ie, logger, None)
|
||||
if not obj.is_available():
|
||||
pytest.skip(f'{obj.PROVIDER_NAME} is not available')
|
||||
obj.is_dev = True
|
||||
return obj
|
||||
|
||||
|
||||
@pytest.mark.download
|
||||
def test_bulk_requests(jcp):
|
||||
assert list(jcp.bulk_solve(requests)) == responses
|
||||
|
||||
|
||||
@pytest.mark.download
|
||||
def test_using_cached_player(jcp):
|
||||
first_player_requests = requests[:3]
|
||||
player = jcp._get_player(first_player_requests[0].video_id, first_player_requests[0].input.player_url)
|
||||
initial = json.loads(jcp._run_js_runtime(jcp._construct_stdin(player, False, first_player_requests)))
|
||||
preprocessed = initial.pop('preprocessed_player')
|
||||
result = json.loads(jcp._run_js_runtime(jcp._construct_stdin(preprocessed, True, first_player_requests)))
|
||||
|
||||
assert initial == result
|
||||
@ -0,0 +1,194 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeRequest,
|
||||
JsChallengeProviderResponse,
|
||||
JsChallengeProviderRejectedRequest,
|
||||
JsChallengeType,
|
||||
JsChallengeResponse,
|
||||
NChallengeOutput,
|
||||
NChallengeInput,
|
||||
JsChallengeProviderError,
|
||||
register_provider,
|
||||
register_preference,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import IEContentProvider
|
||||
from yt_dlp.utils import ExtractorError
|
||||
from yt_dlp.extractor.youtube.jsc._registry import _jsc_preferences, _jsc_providers
|
||||
|
||||
|
||||
class ExampleJCP(JsChallengeProvider):
|
||||
PROVIDER_NAME = 'example-provider'
|
||||
PROVIDER_VERSION = '0.0.1'
|
||||
BUG_REPORT_LOCATION = 'https://example.com/issues'
|
||||
|
||||
_SUPPORTED_TYPES = [JsChallengeType.N]
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def _real_bulk_solve(self, requests):
|
||||
for request in requests:
|
||||
results = dict.fromkeys(request.input.challenges, 'example-solution')
|
||||
response = JsChallengeResponse(
|
||||
type=request.type,
|
||||
output=NChallengeOutput(results=results))
|
||||
yield JsChallengeProviderResponse(request=request, response=response)
|
||||
|
||||
|
||||
PLAYER_URL = 'https://example.com/player.js'
|
||||
|
||||
|
||||
class TestJsChallengeProvider:
|
||||
# note: some test covered in TestPoTokenProvider which shares the same base class
|
||||
def test_base_type(self):
|
||||
assert issubclass(JsChallengeProvider, IEContentProvider)
|
||||
|
||||
def test_create_provider_missing_bulk_solve_method(self, ie, logger):
|
||||
class MissingMethodsJCP(JsChallengeProvider):
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
with pytest.raises(TypeError, match='bulk_solve'):
|
||||
MissingMethodsJCP(ie=ie, logger=logger, settings={})
|
||||
|
||||
def test_create_provider_missing_available_method(self, ie, logger):
|
||||
class MissingMethodsJCP(JsChallengeProvider):
|
||||
def _real_bulk_solve(self, requests):
|
||||
raise JsChallengeProviderRejectedRequest('Not implemented')
|
||||
|
||||
with pytest.raises(TypeError, match='is_available'):
|
||||
MissingMethodsJCP(ie=ie, logger=logger, settings={})
|
||||
|
||||
def test_barebones_provider(self, ie, logger):
|
||||
class BarebonesProviderJCP(JsChallengeProvider):
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def _real_bulk_solve(self, requests):
|
||||
raise JsChallengeProviderRejectedRequest('Not implemented')
|
||||
|
||||
provider = BarebonesProviderJCP(ie=ie, logger=logger, settings={})
|
||||
assert provider.PROVIDER_NAME == 'BarebonesProvider'
|
||||
assert provider.PROVIDER_KEY == 'BarebonesProvider'
|
||||
assert provider.PROVIDER_VERSION == '0.0.0'
|
||||
assert provider.BUG_REPORT_MESSAGE == 'please report this issue to the provider developer at (developer has not provided a bug report location) .'
|
||||
|
||||
def test_example_provider_success(self, ie, logger):
|
||||
provider = ExampleJCP(ie=ie, logger=logger, settings={})
|
||||
|
||||
request = JsChallengeRequest(
|
||||
type=JsChallengeType.N,
|
||||
input=NChallengeInput(player_url=PLAYER_URL, challenges=['example-challenge']))
|
||||
|
||||
request_two = JsChallengeRequest(
|
||||
type=JsChallengeType.N,
|
||||
input=NChallengeInput(player_url=PLAYER_URL, challenges=['example-challenge-2']))
|
||||
|
||||
responses = list(provider.bulk_solve([request, request_two]))
|
||||
assert len(responses) == 2
|
||||
assert all(isinstance(r, JsChallengeProviderResponse) for r in responses)
|
||||
assert responses == [
|
||||
JsChallengeProviderResponse(
|
||||
request=request,
|
||||
response=JsChallengeResponse(
|
||||
type=JsChallengeType.N,
|
||||
output=NChallengeOutput(results={'example-challenge': 'example-solution'}),
|
||||
),
|
||||
),
|
||||
JsChallengeProviderResponse(
|
||||
request=request_two,
|
||||
response=JsChallengeResponse(
|
||||
type=JsChallengeType.N,
|
||||
output=NChallengeOutput(results={'example-challenge-2': 'example-solution'}),
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
def test_provider_unsupported_challenge_type(self, ie, logger):
|
||||
provider = ExampleJCP(ie=ie, logger=logger, settings={})
|
||||
request_supported = JsChallengeRequest(
|
||||
type=JsChallengeType.N,
|
||||
input=NChallengeInput(player_url=PLAYER_URL, challenges=['example-challenge']))
|
||||
request_unsupported = JsChallengeRequest(
|
||||
type=JsChallengeType.SIG,
|
||||
input=NChallengeInput(player_url=PLAYER_URL, challenges=['example-challenge']))
|
||||
responses = list(provider.bulk_solve([request_supported, request_unsupported, request_supported]))
|
||||
assert len(responses) == 3
|
||||
# Requests are validated first before continuing to _real_bulk_solve
|
||||
assert isinstance(responses[0], JsChallengeProviderResponse)
|
||||
assert isinstance(responses[0].error, JsChallengeProviderRejectedRequest)
|
||||
assert responses[0].request is request_unsupported
|
||||
assert str(responses[0].error) == 'JS Challenge type "JsChallengeType.SIG" is not supported by example-provider'
|
||||
|
||||
assert responses[1:] == [
|
||||
JsChallengeProviderResponse(
|
||||
request=request_supported,
|
||||
response=JsChallengeResponse(
|
||||
type=JsChallengeType.N,
|
||||
output=NChallengeOutput(results={'example-challenge': 'example-solution'}),
|
||||
),
|
||||
),
|
||||
JsChallengeProviderResponse(
|
||||
request=request_supported,
|
||||
response=JsChallengeResponse(
|
||||
type=JsChallengeType.N,
|
||||
output=NChallengeOutput(results={'example-challenge': 'example-solution'}),
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
def test_provider_get_player(self, ie, logger):
|
||||
ie._load_player = lambda video_id, player_url, fatal: (video_id, player_url, fatal)
|
||||
provider = ExampleJCP(ie=ie, logger=logger, settings={})
|
||||
assert provider._get_player('video123', PLAYER_URL) == ('video123', PLAYER_URL, True)
|
||||
|
||||
def test_provider_get_player_error(self, ie, logger):
|
||||
def raise_error(video_id, player_url, fatal):
|
||||
raise ExtractorError('Failed to load player')
|
||||
|
||||
ie._load_player = raise_error
|
||||
provider = ExampleJCP(ie=ie, logger=logger, settings={})
|
||||
with pytest.raises(JsChallengeProviderError, match='Failed to load player for JS challenge'):
|
||||
provider._get_player('video123', PLAYER_URL)
|
||||
|
||||
def test_require_class_end_with_suffix(self, ie, logger):
|
||||
class InvalidSuffix(JsChallengeProvider):
|
||||
PROVIDER_NAME = 'invalid-suffix'
|
||||
|
||||
def _real_bulk_solve(self, requests):
|
||||
raise JsChallengeProviderRejectedRequest('Not implemented')
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
provider = InvalidSuffix(ie=ie, logger=logger, settings={})
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
provider.PROVIDER_KEY # noqa: B018
|
||||
|
||||
|
||||
def test_register_provider(ie):
|
||||
|
||||
@register_provider
|
||||
class UnavailableProviderJCP(JsChallengeProvider):
|
||||
def is_available(self) -> bool:
|
||||
return False
|
||||
|
||||
def _real_bulk_solve(self, requests):
|
||||
raise JsChallengeProviderRejectedRequest('Not implemented')
|
||||
|
||||
assert _jsc_providers.value.get('UnavailableProvider') == UnavailableProviderJCP
|
||||
_jsc_providers.value.pop('UnavailableProvider')
|
||||
|
||||
|
||||
def test_register_preference(ie):
|
||||
before = len(_jsc_preferences.value)
|
||||
|
||||
@register_preference(ExampleJCP)
|
||||
def unavailable_preference(*args, **kwargs):
|
||||
return 1
|
||||
|
||||
assert len(_jsc_preferences.value) == before + 1
|
||||
@ -1,504 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
import contextlib
|
||||
import re
|
||||
import string
|
||||
import urllib.request
|
||||
|
||||
from test.helper import FakeYDL, is_download_test
|
||||
from yt_dlp.extractor import YoutubeIE
|
||||
from yt_dlp.jsinterp import JSInterpreter
|
||||
|
||||
_SIG_TESTS = [
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
|
||||
86,
|
||||
'>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
|
||||
85,
|
||||
'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-vfle-mVwz.js',
|
||||
90,
|
||||
']\\[@?>=<;:/.-,+*)(\'&%$#"hZYXWVUTSRQPONMLKJIHGFEDCBAzyxwvutsrqponmlkjiagfedcb39876',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl0Cbn9e.js',
|
||||
84,
|
||||
'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVW@YZ!"#$%&\'()*+,-./:;<=',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js',
|
||||
'2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
|
||||
'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
|
||||
84,
|
||||
'123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js',
|
||||
83,
|
||||
'123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js',
|
||||
'4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
|
||||
'82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B',
|
||||
),
|
||||
(
|
||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
|
||||
'312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
|
||||
'112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/363db69b/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/4fcd6e4a/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/e12fbea4/player_ias.vflset/en_US/base.js',
|
||||
'gN7a-hudCuAuPH6fByOk1_GNXN0yNMHShjZXS2VOgsEItAJz0tipeavEOmNdYN-wUtcEqD3bCXjc0iyKfAyZxCBGgIARwsSdQfJ2CJtt',
|
||||
'JC2JfQdSswRAIgGBCxZyAfKyi0cjXCb3DqEctUw-NYdNmOEvaepit0zJAtIEsgOV2SXZjhSHMNy0NXNG_1kOyBf6HPuAuCduh-a',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/010fbc8d/player_es5.vflset/en_US/base.js',
|
||||
'gN7a-hudCuAuPH6fByOk1_GNXN0yNMHShjZXS2VOgsEItAJz0tipeavEOmNdYN-wUtcEqD3bCXjc0iyKfAyZxCBGgIARwsSdQfJ2CJtt',
|
||||
'ttJC2JfQdSswRAIgGBCxZyAfKyi0cjXCb3DqEctUw-NYdNmOEvaepit2zJAsIEggOVaSXZjhSHMNy0NXNG_1kOyBf6HPuAuCduh-',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/010fbc8d/player_es6.vflset/en_US/base.js',
|
||||
'gN7a-hudCuAuPH6fByOk1_GNXN0yNMHShjZXS2VOgsEItAJz0tipeavEOmNdYN-wUtcEqD3bCXjc0iyKfAyZxCBGgIARwsSdQfJ2CJtt',
|
||||
'ttJC2JfQdSswRAIgGBCxZyAfKyi0cjXCb3DqEctUw-NYdNmOEvaepit2zJAsIEggOVaSXZjhSHMNy0NXNG_1kOyBf6HPuAuCduh-',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/5ec65609/player_ias_tcc.vflset/en_US/base.js',
|
||||
'AAJAJfQdSswRAIgNSN0GDUcHnCIXkKcF61yLBgDHiX1sUhOJdY4_GxunRYCIDeYNYP_16mQTPm5f1OVq3oV1ijUNYPjP4iUSMAjO9bZ',
|
||||
'AJfQdSswRAIgNSN0GDUcHnCIXkKcF61ZLBgDHiX1sUhOJdY4_GxunRYCIDyYNYP_16mQTPm5f1OVq3oV1ijUNYPjP4iUSMAjO9be',
|
||||
),
|
||||
]
|
||||
|
||||
_NSIG_TESTS = [
|
||||
(
|
||||
'https://www.youtube.com/s/player/7862ca1f/player_ias.vflset/en_US/base.js',
|
||||
'X_LCxVDjAavgE5t', 'yxJ1dM6iz5ogUg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
|
||||
'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
|
||||
'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
|
||||
'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
|
||||
'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8040e515/player_ias.vflset/en_US/base.js',
|
||||
'wvOFaY-yjgDuIEg5', 'HkfBFDHmgw4rsw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/e06dea74/player_ias.vflset/en_US/base.js',
|
||||
'AiuodmaDDYw8d3y4bf', 'ankd8eza2T6Qmw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/5dd88d1d/player-plasma-ias-phone-en_US.vflset/base.js',
|
||||
'kSxKFLeqzv_ZyHSAt', 'n8gS8oRlHOxPFA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/324f67b9/player_ias.vflset/en_US/base.js',
|
||||
'xdftNy7dh9QGnhW', '22qLGxrmX8F1rA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
|
||||
'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
|
||||
'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
|
||||
'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/009f1d77/player_ias.vflset/en_US/base.js',
|
||||
'5dwFHw8aFWQUQtffRq', 'audescmLUzI3jw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/en_US/base.js',
|
||||
'5EHDMgYLV6HPGk_Mu-kk', 'n9lUJLHbxUI0GQ',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/113ca41c/player_ias.vflset/en_US/base.js',
|
||||
'cgYl-tlYkhjT7A', 'hI7BBr2zUgcmMg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
|
||||
'M92UUMHa8PdvPd3wyM', '3hPqLJsiNZx7yA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
|
||||
'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/7a062b77/player_ias.vflset/en_US/base.js',
|
||||
'NRcE3y3mVtm_cV-W', 'VbsCYUATvqlt5w',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js',
|
||||
'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/6f20102c/player_ias.vflset/en_US/base.js',
|
||||
'lE8DhoDmKqnmJJ', 'pJTTX6XyJP2BYw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
|
||||
'aCi3iElgd2kq0bxVbQ', 'QX1y8jGb2IbZ0w',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js',
|
||||
'1wWCVpRR96eAmMI87L', 'KSkWAVv1ZQxC3A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js',
|
||||
'_hXMCwMt9qE310D', 'LoZMgkkofRMCZQ',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/590f65a6/player_ias.vflset/en_US/base.js',
|
||||
'1tm7-g_A9zsI8_Lay_', 'xI4Vem4Put_rOg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/b22ef6e7/player_ias.vflset/en_US/base.js',
|
||||
'b6HcntHGkvBLk_FRf', 'kNPW6A7FyP2l8A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/3400486c/player_ias.vflset/en_US/base.js',
|
||||
'lL46g3XifCKUZn1Xfw', 'z767lhet6V2Skl',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20dfca59/player_ias.vflset/en_US/base.js',
|
||||
'-fLCxedkAk4LUTK2', 'O8kfRq1y1eyHGw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/b12cc44b/player_ias.vflset/en_US/base.js',
|
||||
'keLa5R2U00sR9SQK', 'N1OGyujjEwMnLw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js',
|
||||
'gK15nzVyaXE9RsMP3z', 'ZFFWFLPWx9DEgQ',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js',
|
||||
'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js',
|
||||
'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js',
|
||||
'Sy4aDGc0VpYRR9ew_', '5UPOT1VhoZxNLQ',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/d50f54ef/player_ias_tce.vflset/en_US/base.js',
|
||||
'Ha7507LzRmH3Utygtj', 'XFTb2HoeOE5MHg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/074a8365/player_ias_tce.vflset/en_US/base.js',
|
||||
'Ha7507LzRmH3Utygtj', 'ufTsrE0IVYrkl8v',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
|
||||
'N5uAlLqm0eg1GyHO', 'dCBQOejdq5s-ww',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/69f581a5/tv-player-ias.vflset/tv-player-ias.js',
|
||||
'-qIP447rVlTTwaZjY', 'KNcGOksBAvwqQg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js',
|
||||
'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
|
||||
'eWYu5d5YeY_4LyEDc', 'XJQqf-N7Xra3gg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js',
|
||||
'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/4fcd6e4a/player_ias_tce.vflset/en_US/base.js',
|
||||
'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/tv-player-ias.vflset/tv-player-ias.js',
|
||||
'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js',
|
||||
'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js',
|
||||
'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js',
|
||||
'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
|
||||
'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/59b252b9/player_ias.vflset/en_US/base.js',
|
||||
'D3XWVpYgwhLLKNK4AGX', 'aZrQ1qWJ5yv5h',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/fc2a56a5/player_ias.vflset/en_US/base.js',
|
||||
'qTKWg_Il804jd2kAC', 'OtUAm2W6gyzJjB9u',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/fc2a56a5/tv-player-ias.vflset/tv-player-ias.js',
|
||||
'qTKWg_Il804jd2kAC', 'OtUAm2W6gyzJjB9u',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/a74bf670/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', 'hQP7k1hA22OrNTnq',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/6275f73c/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', '-I03XF0iyf6I_X0A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20c72c18/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', '-I03XF0iyf6I_X0A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/9fe2e06e/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', '6r5ekNIiEMPutZy',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/680f8c75/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', '0ml9caTwpa55Jf',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/14397202/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', 'ozZFAN21okDdJTa',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/5dcb2c1f/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', 'p7iTbRZDYAF',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/a10d7fcc/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', '9Zue7DDHJSD',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8e20cb06/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', '5-4tTneTROTpMzba',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/e12fbea4/player_ias_tce.vflset/en_US/base.js',
|
||||
'kM5r52fugSZRAKHfo3', 'XkeRfXIPOkSwfg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/ef259203/player_ias_tce.vflset/en_US/base.js',
|
||||
'rPqBC01nJpqhhi2iA2U', 'hY7dbiKFT51UIA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/010fbc8d/player_es5.vflset/en_US/base.js',
|
||||
'0hlOAlqjFszVvF4Z', 'R-H23bZGAsRFTg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/010fbc8d/player_es6.vflset/en_US/base.js',
|
||||
'0hlOAlqjFszVvF4Z', 'R-H23bZGAsRFTg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/5ec65609/player_ias_tcc.vflset/en_US/base.js',
|
||||
'6l5CTNx4AzIqH4MXM', 'NupToduxHBew1g',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@is_download_test
|
||||
class TestPlayerInfo(unittest.TestCase):
|
||||
def test_youtube_extract_player_info(self):
|
||||
PLAYER_URLS = (
|
||||
('https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js', '4c3f79c5'),
|
||||
('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/en_US/base.js', '64dddad9'),
|
||||
('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/fr_FR/base.js', '64dddad9'),
|
||||
('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),
|
||||
('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-de_DE.vflset/base.js', '64dddad9'),
|
||||
('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-tablet-en_US.vflset/base.js', '64dddad9'),
|
||||
('https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', 'e7567ecf'),
|
||||
('https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', '643afba4'),
|
||||
# obsolete
|
||||
('https://www.youtube.com/yts/jsbin/player_ias-vfle4-e03/en_US/base.js', 'vfle4-e03'),
|
||||
('https://www.youtube.com/yts/jsbin/player_ias-vfl49f_g4/en_US/base.js', 'vfl49f_g4'),
|
||||
('https://www.youtube.com/yts/jsbin/player_ias-vflCPQUIL/en_US/base.js', 'vflCPQUIL'),
|
||||
('https://www.youtube.com/yts/jsbin/player-vflzQZbt7/en_US/base.js', 'vflzQZbt7'),
|
||||
('https://www.youtube.com/yts/jsbin/player-en_US-vflaxXRn1/base.js', 'vflaxXRn1'),
|
||||
('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js', 'vflXGBaUN'),
|
||||
('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', 'vflKjOTVq'),
|
||||
)
|
||||
for player_url, expected_player_id in PLAYER_URLS:
|
||||
player_id = YoutubeIE._extract_player_info(player_url)
|
||||
self.assertEqual(player_id, expected_player_id)
|
||||
|
||||
|
||||
@is_download_test
|
||||
class TestSignature(unittest.TestCase):
|
||||
def setUp(self):
|
||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata/sigs')
|
||||
if not os.path.exists(self.TESTDATA_DIR):
|
||||
os.mkdir(self.TESTDATA_DIR)
|
||||
|
||||
def tearDown(self):
|
||||
with contextlib.suppress(OSError):
|
||||
for f in os.listdir(self.TESTDATA_DIR):
|
||||
os.remove(f)
|
||||
|
||||
|
||||
def t_factory(name, sig_func, url_pattern):
|
||||
def make_tfunc(url, sig_input, expected_sig):
|
||||
m = url_pattern.match(url)
|
||||
assert m, f'{url!r} should follow URL format'
|
||||
test_id = re.sub(r'[/.-]', '_', m.group('id') or m.group('compat_id'))
|
||||
|
||||
def test_func(self):
|
||||
basename = f'player-{test_id}.js'
|
||||
fn = os.path.join(self.TESTDATA_DIR, basename)
|
||||
|
||||
if not os.path.exists(fn):
|
||||
urllib.request.urlretrieve(url, fn)
|
||||
with open(fn, encoding='utf-8') as testf:
|
||||
jscode = testf.read()
|
||||
self.assertEqual(sig_func(jscode, sig_input, url), expected_sig)
|
||||
|
||||
test_func.__name__ = f'test_{name}_js_{test_id}'
|
||||
setattr(TestSignature, test_func.__name__, test_func)
|
||||
return make_tfunc
|
||||
|
||||
|
||||
def signature(jscode, sig_input, player_url):
|
||||
func = YoutubeIE(FakeYDL())._parse_sig_js(jscode, player_url)
|
||||
src_sig = (
|
||||
str(string.printable[:sig_input])
|
||||
if isinstance(sig_input, int) else sig_input)
|
||||
return func(src_sig)
|
||||
|
||||
|
||||
def n_sig(jscode, sig_input, player_url):
|
||||
ie = YoutubeIE(FakeYDL())
|
||||
funcname = ie._extract_n_function_name(jscode, player_url=player_url)
|
||||
jsi = JSInterpreter(jscode)
|
||||
func = jsi.extract_function_from_code(*ie._fixup_n_function_code(*jsi.extract_function_code(funcname), jscode, player_url))
|
||||
return func([sig_input])
|
||||
|
||||
|
||||
make_sig_test = t_factory(
|
||||
'signature', signature,
|
||||
re.compile(r'''(?x)
|
||||
.+(?:
|
||||
/player/(?P<id>[a-zA-Z0-9_/.-]+)|
|
||||
/html5player-(?:en_US-)?(?P<compat_id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?
|
||||
)\.js$'''))
|
||||
for test_spec in _SIG_TESTS:
|
||||
make_sig_test(*test_spec)
|
||||
|
||||
make_nsig_test = t_factory(
|
||||
'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_/.-]+)\.js$'))
|
||||
for test_spec in _NSIG_TESTS:
|
||||
make_nsig_test(*test_spec)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,5 @@
|
||||
# Trigger import of built-in providers
|
||||
from ._builtin.bun import BunJCP as _BunJCP # noqa: F401
|
||||
from ._builtin.deno import DenoJCP as _DenoJCP # noqa: F401
|
||||
from ._builtin.node import NodeJCP as _NodeJCP # noqa: F401
|
||||
from ._builtin.quickjs import QuickJSJCP as _QuickJSJCP # noqa: F401
|
||||
@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import (
|
||||
_EJS_WIKI_URL,
|
||||
EJSBaseJCP,
|
||||
Script,
|
||||
ScriptSource,
|
||||
ScriptType,
|
||||
ScriptVariant,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor import load_script
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils import Popen
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict, clean_proxies
|
||||
|
||||
# KNOWN ISSUES:
|
||||
# - If node_modules is present and includes a requested lib, the version we request is ignored
|
||||
# and whatever installed in node_modules is used.
|
||||
# - No way to ignore existing node_modules, lock files, etc.
|
||||
# - No sandboxing options available
|
||||
# - Cannot detect if npm packages are cached without potentially downloading them.
|
||||
# `--no-install` appears to disable the cache.
|
||||
# - npm auto-install may fail with an integrity error when using HTTP proxies
|
||||
# - npm auto-install HTTP proxy support may be limited on older Bun versions
|
||||
|
||||
|
||||
@register_provider
|
||||
class BunJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'bun'
|
||||
JS_RUNTIME_NAME = 'bun'
|
||||
BUN_NPM_LIB_FILENAME = 'yt.solver.bun.lib.js'
|
||||
SUPPORTED_PROXY_SCHEMES = ['http', 'https']
|
||||
|
||||
def _iter_script_sources(self):
|
||||
yield from super()._iter_script_sources()
|
||||
yield ScriptSource.BUILTIN, self._bun_npm_source
|
||||
|
||||
def _bun_npm_source(self, script_type: ScriptType, /):
|
||||
if script_type != ScriptType.LIB:
|
||||
return None
|
||||
if 'ejs:npm' not in self.ie.get_param('remote_components', []):
|
||||
return self._skip_component('ejs:npm')
|
||||
|
||||
# Check to see if the environment proxies are compatible with Bun npm source
|
||||
if unsupported_scheme := self._check_env_proxies(self._get_env_options()):
|
||||
self.logger.warning(
|
||||
f'Bun NPM package downloads only support HTTP/HTTPS proxies; skipping remote NPM package downloads. '
|
||||
f'Provide another distribution of the challenge solver script or use '
|
||||
f'another JS runtime that supports "{unsupported_scheme}" proxies. '
|
||||
f'For more information and alternatives, refer to {_EJS_WIKI_URL}')
|
||||
return None
|
||||
|
||||
# Bun-specific lib scripts that uses Bun autoimport
|
||||
# https://bun.com/docs/runtime/autoimport
|
||||
error_hook = lambda e: self.logger.warning(
|
||||
f'Failed to read bun challenge solver lib script: {e}{provider_bug_report_message(self)}')
|
||||
code = load_script(
|
||||
self.BUN_NPM_LIB_FILENAME, error_hook=error_hook)
|
||||
if code:
|
||||
return Script(script_type, ScriptVariant.BUN_NPM, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
def _check_env_proxies(self, env):
|
||||
# check that the schemes of both HTTP_PROXY and HTTPS_PROXY are supported
|
||||
for key in ('HTTP_PROXY', 'HTTPS_PROXY'):
|
||||
proxy = env.get(key)
|
||||
if not proxy:
|
||||
continue
|
||||
scheme = urllib.parse.urlparse(proxy).scheme.lower()
|
||||
if scheme not in self.SUPPORTED_PROXY_SCHEMES:
|
||||
return scheme
|
||||
return None
|
||||
|
||||
def _get_env_options(self) -> dict[str, str]:
|
||||
options = os.environ.copy() # pass through existing bun env vars
|
||||
request_proxies = self.ie._downloader.proxies.copy()
|
||||
clean_proxies(request_proxies, HTTPHeaderDict())
|
||||
|
||||
# Apply 'all' proxy first, then allow per-scheme overrides
|
||||
if request_proxies.get('all') is not None:
|
||||
options['HTTP_PROXY'] = options['HTTPS_PROXY'] = request_proxies['all']
|
||||
for key, env in (('http', 'HTTP_PROXY'), ('https', 'HTTPS_PROXY')):
|
||||
val = request_proxies.get(key)
|
||||
if val is not None:
|
||||
options[env] = val
|
||||
if self.ie.get_param('nocheckcertificate'):
|
||||
options['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
|
||||
|
||||
# Prevent segfault: <https://github.com/oven-sh/bun/issues/22901>
|
||||
options.pop('JSC_useJIT', None)
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
options['BUN_JSC_useJIT'] = '0'
|
||||
|
||||
return options
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
# https://bun.com/docs/cli/run
|
||||
options = ['--no-addons', '--prefer-offline']
|
||||
if self._lib_script.variant == ScriptVariant.BUN_NPM:
|
||||
# Enable auto-install even if node_modules is present
|
||||
options.append('--install=fallback')
|
||||
else:
|
||||
options.append('--no-install')
|
||||
cmd = [self.runtime_info.path, '--bun', 'run', *options, '-']
|
||||
self.logger.debug(f'Running bun: {shlex.join(cmd)}')
|
||||
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=self._get_env_options(),
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running bun process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not re.match(r'^Bun v\d+\.\d+\.\d+ \([\w\s]+\)$', line))
|
||||
|
||||
|
||||
@register_preference(BunJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 800
|
||||
@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import (
|
||||
EJSBaseJCP,
|
||||
Script,
|
||||
ScriptSource,
|
||||
ScriptType,
|
||||
ScriptVariant,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor import load_script
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils import Popen, remove_terminal_sequences
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict, clean_proxies
|
||||
|
||||
|
||||
@register_provider
|
||||
class DenoJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'deno'
|
||||
JS_RUNTIME_NAME = 'deno'
|
||||
|
||||
_DENO_BASE_OPTIONS = ['--no-prompt', '--no-remote', '--no-lock', '--node-modules-dir=none', '--no-config']
|
||||
DENO_NPM_LIB_FILENAME = 'yt.solver.deno.lib.js'
|
||||
_NPM_PACKAGES_CACHED = False
|
||||
|
||||
def _iter_script_sources(self):
|
||||
yield from super()._iter_script_sources()
|
||||
yield ScriptSource.BUILTIN, self._deno_npm_source
|
||||
|
||||
def _deno_npm_source(self, script_type: ScriptType, /):
|
||||
if script_type != ScriptType.LIB:
|
||||
return None
|
||||
# Deno-specific lib scripts that use Deno NPM imports
|
||||
error_hook = lambda e: self.logger.warning(
|
||||
f'Failed to read deno challenge solver lib script: {e}{provider_bug_report_message(self)}')
|
||||
code = load_script(
|
||||
self.DENO_NPM_LIB_FILENAME, error_hook=error_hook)
|
||||
if not code:
|
||||
return None
|
||||
if 'ejs:npm' not in self.ie.get_param('remote_components', []):
|
||||
# We may still be able to continue if the npm packages are available/cached
|
||||
self._NPM_PACKAGES_CACHED = self._npm_packages_cached(code)
|
||||
if not self._NPM_PACKAGES_CACHED:
|
||||
return self._skip_component('ejs:npm')
|
||||
return Script(script_type, ScriptVariant.DENO_NPM, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
|
||||
def _npm_packages_cached(self, stdin: str) -> bool:
|
||||
# Check if npm packages are cached, so we can run without --remote-components ejs:npm
|
||||
self.logger.debug('Checking if npm packages are cached')
|
||||
try:
|
||||
self._run_deno(stdin, [*self._DENO_BASE_OPTIONS, '--cached-only'])
|
||||
except JsChallengeProviderError as e:
|
||||
self.logger.trace(f'Deno npm packages not cached: {e}')
|
||||
return False
|
||||
return True
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
options = [*self._DENO_BASE_OPTIONS]
|
||||
if self._lib_script.variant == ScriptVariant.DENO_NPM and self._NPM_PACKAGES_CACHED:
|
||||
options.append('--cached-only')
|
||||
elif self._lib_script.variant != ScriptVariant.DENO_NPM:
|
||||
options.append('--no-npm')
|
||||
options.append('--cached-only')
|
||||
if self.ie.get_param('nocheckcertificate'):
|
||||
options.append('--unsafely-ignore-certificate-errors')
|
||||
# XXX: Convert this extractor-arg into a general option if/when a JSI framework is implemented
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
options.append('--v8-flags=--jitless')
|
||||
return self._run_deno(stdin, options)
|
||||
|
||||
def _get_env_options(self) -> dict[str, str]:
|
||||
options = os.environ.copy() # pass through existing deno env vars
|
||||
request_proxies = self.ie._downloader.proxies.copy()
|
||||
clean_proxies(request_proxies, HTTPHeaderDict())
|
||||
# Apply 'all' proxy first, then allow per-scheme overrides
|
||||
if 'all' in request_proxies and request_proxies['all'] is not None:
|
||||
options['HTTP_PROXY'] = options['HTTPS_PROXY'] = request_proxies['all']
|
||||
for key, env in (('http', 'HTTP_PROXY'), ('https', 'HTTPS_PROXY'), ('no', 'NO_PROXY')):
|
||||
if key in request_proxies and request_proxies[key] is not None:
|
||||
options[env] = request_proxies[key]
|
||||
return options
|
||||
|
||||
def _run_deno(self, stdin, options) -> str:
|
||||
cmd = [self.runtime_info.path, 'run', *options, '-']
|
||||
self.logger.debug(f'Running deno: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=self._get_env_options(),
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running deno process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not (
|
||||
re.match(r'^Download\s+https\S+$', remove_terminal_sequences(line))
|
||||
or re.match(r'DANGER: TLS certificate validation is disabled for all hostnames', remove_terminal_sequences(line))))
|
||||
|
||||
|
||||
@register_preference(DenoJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 1000
|
||||
@ -0,0 +1,326 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import dataclasses
|
||||
import enum
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from yt_dlp.dependencies import yt_dlp_ejs as _has_ejs
|
||||
from yt_dlp.extractor.youtube.jsc._builtin import vendor
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeProviderRejectedRequest,
|
||||
JsChallengeProviderResponse,
|
||||
JsChallengeResponse,
|
||||
JsChallengeType,
|
||||
NChallengeOutput,
|
||||
SigChallengeOutput,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import configuration_arg
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils._jsruntime import JsRuntimeInfo
|
||||
|
||||
if _has_ejs:
|
||||
import yt_dlp_ejs.yt.solver
|
||||
|
||||
TYPE_CHECKING = False
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Generator
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc.provider import JsChallengeRequest
|
||||
|
||||
_EJS_WIKI_URL = 'https://github.com/yt-dlp/yt-dlp/wiki/EJS'
|
||||
|
||||
|
||||
class ScriptType(enum.Enum):
|
||||
LIB = 'lib'
|
||||
CORE = 'core'
|
||||
|
||||
|
||||
class ScriptVariant(enum.Enum):
|
||||
UNKNOWN = 'unknown'
|
||||
MINIFIED = 'minified'
|
||||
UNMINIFIED = 'unminified'
|
||||
DENO_NPM = 'deno_npm'
|
||||
BUN_NPM = 'bun_npm'
|
||||
|
||||
|
||||
class ScriptSource(enum.Enum):
|
||||
PYPACKAGE = 'python package' # PyPI, PyInstaller exe, zipimport binary, etc
|
||||
CACHE = 'cache' # GitHub release assets (cached)
|
||||
WEB = 'web' # GitHub release assets (downloaded)
|
||||
BUILTIN = 'builtin' # vendored (full core script; import-only lib script + NPM cache)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Script:
|
||||
type: ScriptType
|
||||
variant: ScriptVariant
|
||||
source: ScriptSource
|
||||
version: str
|
||||
code: str
|
||||
|
||||
@functools.cached_property
|
||||
def hash(self, /) -> str:
|
||||
return hashlib.sha3_512(self.code.encode()).hexdigest()
|
||||
|
||||
def __str__(self, /):
|
||||
return f'<Script {self.type.value!r} v{self.version} (source: {self.source.value}) variant={self.variant.value!r} size={len(self.code)} hash={self.hash[:7]}...>'
|
||||
|
||||
|
||||
class EJSBaseJCP(JsChallengeProvider):
|
||||
JS_RUNTIME_NAME: str
|
||||
_CACHE_SECTION = 'challenge-solver'
|
||||
|
||||
_REPOSITORY = 'yt-dlp/ejs'
|
||||
_SUPPORTED_TYPES = [JsChallengeType.N, JsChallengeType.SIG]
|
||||
_SCRIPT_VERSION = vendor.VERSION
|
||||
# TODO: Integration tests for each kind of scripts source
|
||||
_ALLOWED_HASHES = {
|
||||
ScriptType.LIB: {
|
||||
ScriptVariant.UNMINIFIED: vendor.HASHES['yt.solver.lib.js'],
|
||||
ScriptVariant.MINIFIED: vendor.HASHES['yt.solver.lib.min.js'],
|
||||
ScriptVariant.DENO_NPM: vendor.HASHES['yt.solver.deno.lib.js'],
|
||||
ScriptVariant.BUN_NPM: vendor.HASHES['yt.solver.bun.lib.js'],
|
||||
},
|
||||
ScriptType.CORE: {
|
||||
ScriptVariant.MINIFIED: vendor.HASHES['yt.solver.core.min.js'],
|
||||
ScriptVariant.UNMINIFIED: vendor.HASHES['yt.solver.core.js'],
|
||||
},
|
||||
}
|
||||
|
||||
_SCRIPT_FILENAMES = {
|
||||
ScriptType.LIB: 'yt.solver.lib.js',
|
||||
ScriptType.CORE: 'yt.solver.core.js',
|
||||
}
|
||||
|
||||
_MIN_SCRIPT_FILENAMES = {
|
||||
ScriptType.LIB: 'yt.solver.lib.min.js',
|
||||
ScriptType.CORE: 'yt.solver.core.min.js',
|
||||
}
|
||||
|
||||
# currently disabled as files are large and we do not support rotation
|
||||
_ENABLE_PREPROCESSED_PLAYER_CACHE = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._available = True
|
||||
self.ejs_settings = self.ie.get_param('extractor_args', {}).get('youtube-ejs', {})
|
||||
|
||||
# Note: The following 3 args are for developer use only & intentionally not documented.
|
||||
# - dev: bypasses verification of script hashes and versions.
|
||||
# - repo: use a custom GitHub repository to fetch web script from.
|
||||
# - script_version: use a custom script version.
|
||||
# E.g. --extractor-args "youtube-ejs:dev=true;script_version=0.1.4"
|
||||
|
||||
self.is_dev = self.ejs_setting('dev', ['false'])[0] == 'true'
|
||||
if self.is_dev:
|
||||
self.report_dev_option('You have enabled dev mode for EJS JCP Providers.')
|
||||
|
||||
custom_repo = self.ejs_setting('repo', [None])[0]
|
||||
if custom_repo:
|
||||
self.report_dev_option(f'You have set a custom GitHub repository for EJS JCP Providers ({custom_repo}).')
|
||||
self._REPOSITORY = custom_repo
|
||||
|
||||
custom_version = self.ejs_setting('script_version', [None])[0]
|
||||
if custom_version:
|
||||
self.report_dev_option(f'You have set a custom EJS script version for EJS JCP Providers ({custom_version}).')
|
||||
self._SCRIPT_VERSION = custom_version
|
||||
|
||||
def ejs_setting(self, key, *args, **kwargs):
|
||||
return configuration_arg(self.ejs_settings, key, *args, **kwargs)
|
||||
|
||||
def report_dev_option(self, message: str):
|
||||
self.ie.report_warning(
|
||||
f'{message} '
|
||||
f'This is a developer option intended for debugging. \n'
|
||||
' If you experience any issues while using this option, '
|
||||
f'{self.ie._downloader._format_err("DO NOT", self.ie._downloader.Styles.ERROR)} open a bug report', only_once=True)
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
"""To be implemented by subclasses"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _real_bulk_solve(self, /, requests: list[JsChallengeRequest]):
|
||||
grouped: dict[str, list[JsChallengeRequest]] = collections.defaultdict(list)
|
||||
for request in requests:
|
||||
grouped[request.input.player_url].append(request)
|
||||
|
||||
for player_url, grouped_requests in grouped.items():
|
||||
player = None
|
||||
if self._ENABLE_PREPROCESSED_PLAYER_CACHE:
|
||||
player = self.ie.cache.load(self._CACHE_SECTION, f'player:{player_url}')
|
||||
|
||||
if player:
|
||||
cached = True
|
||||
else:
|
||||
cached = False
|
||||
video_id = next((request.video_id for request in grouped_requests), None)
|
||||
player = self._get_player(video_id, player_url)
|
||||
|
||||
# NB: This output belongs after the player request
|
||||
self.logger.info(f'Solving JS challenges using {self.JS_RUNTIME_NAME}')
|
||||
|
||||
stdin = self._construct_stdin(player, cached, grouped_requests)
|
||||
stdout = self._run_js_runtime(stdin)
|
||||
output = json.loads(stdout)
|
||||
if output['type'] == 'error':
|
||||
raise JsChallengeProviderError(output['error'])
|
||||
|
||||
if self._ENABLE_PREPROCESSED_PLAYER_CACHE and (preprocessed := output.get('preprocessed_player')):
|
||||
self.ie.cache.store(self._CACHE_SECTION, f'player:{player_url}', preprocessed)
|
||||
|
||||
for request, response_data in zip(grouped_requests, output['responses'], strict=True):
|
||||
if response_data['type'] == 'error':
|
||||
yield JsChallengeProviderResponse(request, None, response_data['error'])
|
||||
else:
|
||||
yield JsChallengeProviderResponse(request, JsChallengeResponse(request.type, (
|
||||
NChallengeOutput(response_data['data']) if request.type is JsChallengeType.N
|
||||
else SigChallengeOutput(response_data['data']))))
|
||||
|
||||
def _construct_stdin(self, player: str, preprocessed: bool, requests: list[JsChallengeRequest], /) -> str:
|
||||
json_requests = [{
|
||||
'type': request.type.value,
|
||||
'challenges': request.input.challenges,
|
||||
} for request in requests]
|
||||
data = {
|
||||
'type': 'preprocessed',
|
||||
'preprocessed_player': player,
|
||||
'requests': json_requests,
|
||||
} if preprocessed else {
|
||||
'type': 'player',
|
||||
'player': player,
|
||||
'requests': json_requests,
|
||||
'output_preprocessed': True,
|
||||
}
|
||||
return f'''\
|
||||
{self._lib_script.code}
|
||||
Object.assign(globalThis, lib);
|
||||
{self._core_script.code}
|
||||
console.log(JSON.stringify(jsc({json.dumps(data)})));
|
||||
'''
|
||||
|
||||
# region: challenge solver script
|
||||
|
||||
@functools.cached_property
|
||||
def _lib_script(self, /):
|
||||
return self._get_script(ScriptType.LIB)
|
||||
|
||||
@functools.cached_property
|
||||
def _core_script(self, /):
|
||||
return self._get_script(ScriptType.CORE)
|
||||
|
||||
def _get_script(self, script_type: ScriptType, /) -> Script:
|
||||
skipped_components: list[_SkippedComponent] = []
|
||||
for _, from_source in self._iter_script_sources():
|
||||
script = from_source(script_type)
|
||||
if not script:
|
||||
continue
|
||||
if isinstance(script, _SkippedComponent):
|
||||
skipped_components.append(script)
|
||||
continue
|
||||
if not self.is_dev:
|
||||
if script.version != self._SCRIPT_VERSION:
|
||||
self.logger.warning(
|
||||
f'Challenge solver {script_type.value} script version {script.version} '
|
||||
f'is not supported (source: {script.source.value}, variant: {script.variant}, supported version: {self._SCRIPT_VERSION})')
|
||||
if script.source is ScriptSource.CACHE:
|
||||
self.logger.debug('Clearing outdated cached script')
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, None)
|
||||
continue
|
||||
script_hashes = self._ALLOWED_HASHES[script.type].get(script.variant, [])
|
||||
if script_hashes and script.hash not in script_hashes:
|
||||
self.logger.warning(
|
||||
f'Hash mismatch on challenge solver {script.type.value} script '
|
||||
f'(source: {script.source.value}, variant: {script.variant}, hash: {script.hash})!{provider_bug_report_message(self)}')
|
||||
if script.source is ScriptSource.CACHE:
|
||||
self.logger.debug('Clearing invalid cached script')
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, None)
|
||||
continue
|
||||
self.logger.debug(
|
||||
f'Using challenge solver {script.type.value} script v{script.version} '
|
||||
f'(source: {script.source.value}, variant: {script.variant.value})')
|
||||
break
|
||||
|
||||
else:
|
||||
self._available = False
|
||||
raise JsChallengeProviderRejectedRequest(
|
||||
f'No usable challenge solver {script_type.value} script available',
|
||||
_skipped_components=skipped_components or None,
|
||||
)
|
||||
|
||||
return script
|
||||
|
||||
def _iter_script_sources(self) -> Generator[tuple[ScriptSource, Callable[[ScriptType], Script | None]]]:
|
||||
yield from [
|
||||
(ScriptSource.PYPACKAGE, self._pypackage_source),
|
||||
(ScriptSource.CACHE, self._cached_source),
|
||||
(ScriptSource.BUILTIN, self._builtin_source),
|
||||
(ScriptSource.WEB, self._web_release_source)]
|
||||
|
||||
def _pypackage_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
if not _has_ejs:
|
||||
return None
|
||||
try:
|
||||
code = yt_dlp_ejs.yt.solver.core() if script_type is ScriptType.CORE else yt_dlp_ejs.yt.solver.lib()
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f'Failed to load challenge solver {script_type.value} script from python package: {e}{provider_bug_report_message(self)}')
|
||||
return None
|
||||
return Script(script_type, ScriptVariant.MINIFIED, ScriptSource.PYPACKAGE, yt_dlp_ejs.version, code)
|
||||
|
||||
def _cached_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
if data := self.ie.cache.load(self._CACHE_SECTION, script_type.value):
|
||||
return Script(script_type, ScriptVariant(data['variant']), ScriptSource.CACHE, data['version'], data['code'])
|
||||
return None
|
||||
|
||||
def _builtin_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
error_hook = lambda _: self.logger.warning(
|
||||
f'Failed to read builtin challenge solver {script_type.value} script{provider_bug_report_message(self)}')
|
||||
code = vendor.load_script(
|
||||
self._SCRIPT_FILENAMES[script_type], error_hook=error_hook)
|
||||
if code:
|
||||
return Script(script_type, ScriptVariant.UNMINIFIED, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
def _web_release_source(self, script_type: ScriptType, /):
|
||||
if 'ejs:github' not in (self.ie.get_param('remote_components') or ()):
|
||||
return self._skip_component('ejs:github')
|
||||
url = f'https://github.com/{self._REPOSITORY}/releases/download/{self._SCRIPT_VERSION}/{self._MIN_SCRIPT_FILENAMES[script_type]}'
|
||||
if code := self.ie._download_webpage_with_retries(
|
||||
url, None, f'[{self.logger.prefix}] Downloading challenge solver {script_type.value} script from {url}',
|
||||
f'[{self.logger.prefix}] Failed to download challenge solver {script_type.value} script', fatal=False,
|
||||
):
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, {
|
||||
'version': self._SCRIPT_VERSION,
|
||||
'variant': ScriptVariant.MINIFIED.value,
|
||||
'code': code,
|
||||
})
|
||||
return Script(script_type, ScriptVariant.MINIFIED, ScriptSource.WEB, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
# endregion: challenge solver script
|
||||
|
||||
@property
|
||||
def runtime_info(self) -> JsRuntimeInfo | None:
|
||||
runtime = self.ie._downloader._js_runtimes.get(self.JS_RUNTIME_NAME)
|
||||
if not runtime or not runtime.info or not runtime.info.supported:
|
||||
return None
|
||||
return runtime.info
|
||||
|
||||
def is_available(self, /) -> bool:
|
||||
if not self.runtime_info:
|
||||
return False
|
||||
return self._available
|
||||
|
||||
def _skip_component(self, component: str, /):
|
||||
return _SkippedComponent(component, self.JS_RUNTIME_NAME)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _SkippedComponent:
|
||||
component: str
|
||||
runtime: str
|
||||
@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import EJSBaseJCP
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.utils import Popen
|
||||
|
||||
|
||||
@register_provider
|
||||
class NodeJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'node'
|
||||
JS_RUNTIME_NAME = 'node'
|
||||
|
||||
_ARGS = ['-']
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
args = []
|
||||
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
args.append('--v8-flags=--jitless')
|
||||
|
||||
# Node permission flag changed from experimental to stable in v23.5.0
|
||||
if self.runtime_info.version_tuple < (23, 5, 0):
|
||||
args.append('--experimental-permission')
|
||||
args.append('--no-warnings=ExperimentalWarning')
|
||||
else:
|
||||
args.append('--permission')
|
||||
|
||||
cmd = [self.runtime_info.path, *args, *self._ARGS]
|
||||
self.logger.debug(f'Running node: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running node process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not (
|
||||
re.match(r'^\[stdin\]:', line)
|
||||
or re.match(r'^var jsc', line)
|
||||
or '(Use `node --trace-uncaught ...` to show where the exception was thrown)' == line
|
||||
or re.match(r'^Node\.js v\d+\.\d+\.\d+$', line)))
|
||||
|
||||
|
||||
@register_preference(NodeJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 900
|
||||
@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import EJSBaseJCP
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.utils import Popen
|
||||
|
||||
|
||||
@register_provider
|
||||
class QuickJSJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'quickjs'
|
||||
JS_RUNTIME_NAME = 'quickjs'
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
if self.runtime_info.name == 'quickjs-ng':
|
||||
self.logger.warning('QuickJS-NG is missing some optimizations making this very slow. Consider using upstream QuickJS instead.')
|
||||
elif self.runtime_info.version_tuple < (2025, 4, 26):
|
||||
self.logger.warning('Older QuickJS versions are missing optimizations making this very slow. Consider upgrading.')
|
||||
|
||||
# QuickJS does not support reading from stdin, so we have to use a temp file
|
||||
temp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.js', delete=False, encoding='utf-8')
|
||||
try:
|
||||
temp_file.write(stdin)
|
||||
temp_file.close()
|
||||
cmd = [self.runtime_info.path, '--script', temp_file.name]
|
||||
self.logger.debug(f'Running QuickJS: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running QuickJS process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
finally:
|
||||
pathlib.Path(temp_file.name).unlink(missing_ok=True)
|
||||
|
||||
return stdout
|
||||
|
||||
|
||||
@register_preference(QuickJSJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 850
|
||||
@ -0,0 +1,17 @@
|
||||
import importlib.resources
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor._info import HASHES, VERSION
|
||||
|
||||
__all__ = ['HASHES', 'VERSION', 'load_script']
|
||||
|
||||
|
||||
def load_script(filename, error_hook=None):
|
||||
file = importlib.resources.files(__package__) / filename
|
||||
if file.is_file():
|
||||
try:
|
||||
return file.read_text(encoding='utf-8')
|
||||
except (OSError, FileNotFoundError, ModuleNotFoundError) as e:
|
||||
if error_hook:
|
||||
error_hook(e)
|
||||
return None
|
||||
return None
|
||||
@ -0,0 +1,11 @@
|
||||
# This file is generated by devscripts/update_ejs.py. DO NOT MODIFY!
|
||||
|
||||
VERSION = '0.3.0'
|
||||
HASHES = {
|
||||
'yt.solver.bun.lib.js': '6ff45e94de9f0ea936a183c48173cfa9ce526ee4b7544cd556428427c1dd53c8073ef0174e79b320252bf0e7c64b0032cc1cf9c4358f3fda59033b7caa01c241',
|
||||
'yt.solver.core.js': '0cd96b2d3f319dfa62cae689efa7d930ef1706e95f5921794db5089b2262957ec0a17d73938d8975ea35d0309cbfb4c8e4418d5e219837215eee242890c8b64d',
|
||||
'yt.solver.core.min.js': '370d627703002b4a73b10027702734a3de9484f6b56b739942be1dc2b60fee49dee2aa86ed117d1c8ae1ac55181d326481f1fe2e2e8d5211154d48e2a55dac51',
|
||||
'yt.solver.deno.lib.js': '9c8ee3ab6c23e443a5a951e3ac73c6b8c1c8fb34335e7058a07bf99d349be5573611de00536dcd03ecd3cf34014c4e9b536081de37af3637c5390c6a6fd6a0f0',
|
||||
'yt.solver.lib.js': '1ee3753a8222fc855f5c39db30a9ccbb7967dbe1fb810e86dc9a89aa073a0907f294c720e9b65427d560a35aa1ce6af19ef854d9126a05ca00afe03f72047733',
|
||||
'yt.solver.lib.min.js': '8420c259ad16e99ce004e4651ac1bcabb53b4457bf5668a97a9359be9a998a789fee8ab124ee17f91a2ea8fd84e0f2b2fc8eabcaf0b16a186ba734cf422ad053',
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
const lib = {
|
||||
meriyah: await import('meriyah@6.1.4'),
|
||||
astring: await import('astring@1.9.0'),
|
||||
};
|
||||
export { lib };
|
||||
@ -0,0 +1,550 @@
|
||||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
var jsc = (function (meriyah, astring) {
|
||||
'use strict';
|
||||
function matchesStructure(obj, structure) {
|
||||
if (Array.isArray(structure)) {
|
||||
if (!Array.isArray(obj)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
structure.length === obj.length &&
|
||||
structure.every((value, index) => matchesStructure(obj[index], value))
|
||||
);
|
||||
}
|
||||
if (typeof structure === 'object') {
|
||||
if (!obj) {
|
||||
return !structure;
|
||||
}
|
||||
if ('or' in structure) {
|
||||
return structure.or.some((node) => matchesStructure(obj, node));
|
||||
}
|
||||
if ('anykey' in structure && Array.isArray(structure.anykey)) {
|
||||
const haystack = Array.isArray(obj) ? obj : Object.values(obj);
|
||||
return structure.anykey.every((value) =>
|
||||
haystack.some((el) => matchesStructure(el, value)),
|
||||
);
|
||||
}
|
||||
for (const [key, value] of Object.entries(structure)) {
|
||||
if (!matchesStructure(obj[key], value)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return structure === obj;
|
||||
}
|
||||
function isOneOf(value, ...of) {
|
||||
return of.includes(value);
|
||||
}
|
||||
function _optionalChain$2(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
const logicalExpression = {
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'LogicalExpression',
|
||||
left: { type: 'Identifier' },
|
||||
right: {
|
||||
type: 'SequenceExpression',
|
||||
expressions: [
|
||||
{
|
||||
type: 'AssignmentExpression',
|
||||
left: { type: 'Identifier' },
|
||||
operator: '=',
|
||||
right: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier' },
|
||||
arguments: {
|
||||
or: [
|
||||
[
|
||||
{ type: 'Literal' },
|
||||
{
|
||||
type: 'CallExpression',
|
||||
callee: {
|
||||
type: 'Identifier',
|
||||
name: 'decodeURIComponent',
|
||||
},
|
||||
arguments: [{ type: 'Identifier' }],
|
||||
optional: false,
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
type: 'CallExpression',
|
||||
callee: {
|
||||
type: 'Identifier',
|
||||
name: 'decodeURIComponent',
|
||||
},
|
||||
arguments: [{ type: 'Identifier' }],
|
||||
optional: false,
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
optional: false,
|
||||
},
|
||||
},
|
||||
{ type: 'CallExpression' },
|
||||
],
|
||||
},
|
||||
operator: '&&',
|
||||
},
|
||||
};
|
||||
const identifier$1 = {
|
||||
or: [
|
||||
{
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
operator: '=',
|
||||
left: { type: 'Identifier' },
|
||||
right: { type: 'FunctionExpression', params: [{}, {}, {}] },
|
||||
},
|
||||
},
|
||||
{ type: 'FunctionDeclaration', params: [{}, {}, {}] },
|
||||
{
|
||||
type: 'VariableDeclaration',
|
||||
declarations: {
|
||||
anykey: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
init: { type: 'FunctionExpression', params: [{}, {}, {}] },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
function extract$1(node) {
|
||||
if (!matchesStructure(node, identifier$1)) {
|
||||
return null;
|
||||
}
|
||||
let block;
|
||||
if (
|
||||
node.type === 'ExpressionStatement' &&
|
||||
node.expression.type === 'AssignmentExpression' &&
|
||||
node.expression.right.type === 'FunctionExpression'
|
||||
) {
|
||||
block = node.expression.right.body;
|
||||
} else if (node.type === 'VariableDeclaration') {
|
||||
for (const decl of node.declarations) {
|
||||
if (
|
||||
decl.type === 'VariableDeclarator' &&
|
||||
_optionalChain$2([
|
||||
decl,
|
||||
'access',
|
||||
(_) => _.init,
|
||||
'optionalAccess',
|
||||
(_2) => _2.type,
|
||||
]) === 'FunctionExpression' &&
|
||||
_optionalChain$2([
|
||||
decl,
|
||||
'access',
|
||||
(_3) => _3.init,
|
||||
'optionalAccess',
|
||||
(_4) => _4.params,
|
||||
'access',
|
||||
(_5) => _5.length,
|
||||
]) === 3
|
||||
) {
|
||||
block = decl.init.body;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'FunctionDeclaration') {
|
||||
block = node.body;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
const relevantExpression = _optionalChain$2([
|
||||
block,
|
||||
'optionalAccess',
|
||||
(_6) => _6.body,
|
||||
'access',
|
||||
(_7) => _7.at,
|
||||
'call',
|
||||
(_8) => _8(-2),
|
||||
]);
|
||||
if (!matchesStructure(relevantExpression, logicalExpression)) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
_optionalChain$2([
|
||||
relevantExpression,
|
||||
'optionalAccess',
|
||||
(_9) => _9.type,
|
||||
]) !== 'ExpressionStatement' ||
|
||||
relevantExpression.expression.type !== 'LogicalExpression' ||
|
||||
relevantExpression.expression.right.type !== 'SequenceExpression' ||
|
||||
relevantExpression.expression.right.expressions[0].type !==
|
||||
'AssignmentExpression'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const call = relevantExpression.expression.right.expressions[0].right;
|
||||
if (call.type !== 'CallExpression' || call.callee.type !== 'Identifier') {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: 'ArrowFunctionExpression',
|
||||
params: [{ type: 'Identifier', name: 'sig' }],
|
||||
body: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier', name: call.callee.name },
|
||||
arguments:
|
||||
call.arguments.length === 1
|
||||
? [{ type: 'Identifier', name: 'sig' }]
|
||||
: [call.arguments[0], { type: 'Identifier', name: 'sig' }],
|
||||
optional: false,
|
||||
},
|
||||
async: false,
|
||||
expression: false,
|
||||
generator: false,
|
||||
};
|
||||
}
|
||||
function _optionalChain$1(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
const identifier = {
|
||||
or: [
|
||||
{
|
||||
type: 'VariableDeclaration',
|
||||
kind: 'var',
|
||||
declarations: {
|
||||
anykey: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
id: { type: 'Identifier' },
|
||||
init: {
|
||||
type: 'ArrayExpression',
|
||||
elements: [{ type: 'Identifier' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
left: { type: 'Identifier' },
|
||||
operator: '=',
|
||||
right: {
|
||||
type: 'ArrayExpression',
|
||||
elements: [{ type: 'Identifier' }],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const catchBlockBody = [
|
||||
{
|
||||
type: 'ReturnStatement',
|
||||
argument: {
|
||||
type: 'BinaryExpression',
|
||||
left: {
|
||||
type: 'MemberExpression',
|
||||
object: { type: 'Identifier' },
|
||||
computed: true,
|
||||
property: { type: 'Literal' },
|
||||
optional: false,
|
||||
},
|
||||
right: { type: 'Identifier' },
|
||||
operator: '+',
|
||||
},
|
||||
},
|
||||
];
|
||||
function extract(node) {
|
||||
if (!matchesStructure(node, identifier)) {
|
||||
let name = null;
|
||||
let block = null;
|
||||
switch (node.type) {
|
||||
case 'ExpressionStatement': {
|
||||
if (
|
||||
node.expression.type === 'AssignmentExpression' &&
|
||||
node.expression.left.type === 'Identifier' &&
|
||||
node.expression.right.type === 'FunctionExpression' &&
|
||||
node.expression.right.params.length === 1
|
||||
) {
|
||||
name = node.expression.left.name;
|
||||
block = node.expression.right.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'FunctionDeclaration': {
|
||||
if (node.params.length === 1) {
|
||||
name = _optionalChain$1([
|
||||
node,
|
||||
'access',
|
||||
(_) => _.id,
|
||||
'optionalAccess',
|
||||
(_2) => _2.name,
|
||||
]);
|
||||
block = node.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!block || !name) {
|
||||
return null;
|
||||
}
|
||||
const tryNode = block.body.at(-2);
|
||||
if (
|
||||
_optionalChain$1([tryNode, 'optionalAccess', (_3) => _3.type]) !==
|
||||
'TryStatement' ||
|
||||
_optionalChain$1([
|
||||
tryNode,
|
||||
'access',
|
||||
(_4) => _4.handler,
|
||||
'optionalAccess',
|
||||
(_5) => _5.type,
|
||||
]) !== 'CatchClause'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const catchBody = tryNode.handler.body.body;
|
||||
if (matchesStructure(catchBody, catchBlockBody)) {
|
||||
return makeSolverFuncFromName(name);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (node.type === 'VariableDeclaration') {
|
||||
for (const declaration of node.declarations) {
|
||||
if (
|
||||
declaration.type !== 'VariableDeclarator' ||
|
||||
!declaration.init ||
|
||||
declaration.init.type !== 'ArrayExpression' ||
|
||||
declaration.init.elements.length !== 1
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const [firstElement] = declaration.init.elements;
|
||||
if (firstElement && firstElement.type === 'Identifier') {
|
||||
return makeSolverFuncFromName(firstElement.name);
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'ExpressionStatement') {
|
||||
const expr = node.expression;
|
||||
if (
|
||||
expr.type === 'AssignmentExpression' &&
|
||||
expr.left.type === 'Identifier' &&
|
||||
expr.operator === '=' &&
|
||||
expr.right.type === 'ArrayExpression' &&
|
||||
expr.right.elements.length === 1
|
||||
) {
|
||||
const [firstElement] = expr.right.elements;
|
||||
if (firstElement && firstElement.type === 'Identifier') {
|
||||
return makeSolverFuncFromName(firstElement.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function makeSolverFuncFromName(name) {
|
||||
return {
|
||||
type: 'ArrowFunctionExpression',
|
||||
params: [{ type: 'Identifier', name: 'n' }],
|
||||
body: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier', name: name },
|
||||
arguments: [{ type: 'Identifier', name: 'n' }],
|
||||
optional: false,
|
||||
},
|
||||
async: false,
|
||||
expression: false,
|
||||
generator: false,
|
||||
};
|
||||
}
|
||||
const setupNodes = meriyah.parse(
|
||||
`\nif (typeof globalThis.XMLHttpRequest === "undefined") {\n globalThis.XMLHttpRequest = { prototype: {} };\n}\nconst window = Object.create(null);\nif (typeof URL === "undefined") {\n window.location = {\n hash: "",\n host: "www.youtube.com",\n hostname: "www.youtube.com",\n href: "https://www.youtube.com/watch?v=yt-dlp-wins",\n origin: "https://www.youtube.com",\n password: "",\n pathname: "/watch",\n port: "",\n protocol: "https:",\n search: "?v=yt-dlp-wins",\n username: "",\n };\n} else {\n window.location = new URL("https://www.youtube.com/watch?v=yt-dlp-wins");\n}\nif (typeof globalThis.document === "undefined") {\n globalThis.document = Object.create(null);\n}\nif (typeof globalThis.navigator === "undefined") {\n globalThis.navigator = Object.create(null);\n}\nif (typeof globalThis.self === "undefined") {\n globalThis.self = globalThis;\n}\n`,
|
||||
).body;
|
||||
function _optionalChain(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function preprocessPlayer(data) {
|
||||
const ast = meriyah.parse(data);
|
||||
const body = ast.body;
|
||||
const block = (() => {
|
||||
switch (body.length) {
|
||||
case 1: {
|
||||
const func = body[0];
|
||||
if (
|
||||
_optionalChain([func, 'optionalAccess', (_) => _.type]) ===
|
||||
'ExpressionStatement' &&
|
||||
func.expression.type === 'CallExpression' &&
|
||||
func.expression.callee.type === 'MemberExpression' &&
|
||||
func.expression.callee.object.type === 'FunctionExpression'
|
||||
) {
|
||||
return func.expression.callee.object.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
const func = body[1];
|
||||
if (
|
||||
_optionalChain([func, 'optionalAccess', (_2) => _2.type]) ===
|
||||
'ExpressionStatement' &&
|
||||
func.expression.type === 'CallExpression' &&
|
||||
func.expression.callee.type === 'FunctionExpression'
|
||||
) {
|
||||
const block = func.expression.callee.body;
|
||||
block.body.splice(0, 1);
|
||||
return block;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
throw 'unexpected structure';
|
||||
})();
|
||||
const found = { n: [], sig: [] };
|
||||
const plainExpressions = block.body.filter((node) => {
|
||||
const n = extract(node);
|
||||
if (n) {
|
||||
found.n.push(n);
|
||||
}
|
||||
const sig = extract$1(node);
|
||||
if (sig) {
|
||||
found.sig.push(sig);
|
||||
}
|
||||
if (node.type === 'ExpressionStatement') {
|
||||
if (node.expression.type === 'AssignmentExpression') {
|
||||
return true;
|
||||
}
|
||||
return node.expression.type === 'Literal';
|
||||
}
|
||||
return true;
|
||||
});
|
||||
block.body = plainExpressions;
|
||||
for (const [name, options] of Object.entries(found)) {
|
||||
const unique = new Set(options.map((x) => JSON.stringify(x)));
|
||||
if (unique.size !== 1) {
|
||||
const message = `found ${unique.size} ${name} function possibilities`;
|
||||
throw (
|
||||
message +
|
||||
(unique.size
|
||||
? `: ${options.map((x) => astring.generate(x)).join(', ')}`
|
||||
: '')
|
||||
);
|
||||
}
|
||||
plainExpressions.push({
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
operator: '=',
|
||||
left: {
|
||||
type: 'MemberExpression',
|
||||
computed: false,
|
||||
object: { type: 'Identifier', name: '_result' },
|
||||
property: { type: 'Identifier', name: name },
|
||||
},
|
||||
right: options[0],
|
||||
},
|
||||
});
|
||||
}
|
||||
ast.body.splice(0, 0, ...setupNodes);
|
||||
return astring.generate(ast);
|
||||
}
|
||||
function getFromPrepared(code) {
|
||||
const resultObj = { n: null, sig: null };
|
||||
Function('_result', code)(resultObj);
|
||||
return resultObj;
|
||||
}
|
||||
function main(input) {
|
||||
const preprocessedPlayer =
|
||||
input.type === 'player'
|
||||
? preprocessPlayer(input.player)
|
||||
: input.preprocessed_player;
|
||||
const solvers = getFromPrepared(preprocessedPlayer);
|
||||
const responses = input.requests.map((input) => {
|
||||
if (!isOneOf(input.type, 'n', 'sig')) {
|
||||
return { type: 'error', error: `Unknown request type: ${input.type}` };
|
||||
}
|
||||
const solver = solvers[input.type];
|
||||
if (!solver) {
|
||||
return {
|
||||
type: 'error',
|
||||
error: `Failed to extract ${input.type} function`,
|
||||
};
|
||||
}
|
||||
try {
|
||||
return {
|
||||
type: 'result',
|
||||
data: Object.fromEntries(
|
||||
input.challenges.map((challenge) => [challenge, solver(challenge)]),
|
||||
),
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
type: 'error',
|
||||
error:
|
||||
error instanceof Error
|
||||
? `${error.message}\n${error.stack}`
|
||||
: `${error}`,
|
||||
};
|
||||
}
|
||||
});
|
||||
const output = { type: 'result', responses: responses };
|
||||
if (input.type === 'player' && input.output_preprocessed) {
|
||||
output.preprocessed_player = preprocessedPlayer;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
return main;
|
||||
})(meriyah, astring);
|
||||
@ -0,0 +1,9 @@
|
||||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
const lib = {
|
||||
meriyah: await import('npm:meriyah@6.1.4'),
|
||||
astring: await import('npm:astring@1.9.0'),
|
||||
};
|
||||
export { lib };
|
||||
@ -0,0 +1,287 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import _EJS_WIKI_URL
|
||||
from yt_dlp.extractor.youtube.jsc._registry import (
|
||||
_jsc_preferences,
|
||||
_jsc_providers,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeProviderRejectedRequest,
|
||||
JsChallengeProviderResponse,
|
||||
JsChallengeRequest,
|
||||
JsChallengeResponse,
|
||||
JsChallengeType,
|
||||
NChallengeInput,
|
||||
NChallengeOutput,
|
||||
SigChallengeInput,
|
||||
SigChallengeOutput,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._director import YoutubeIEContentProviderLogger, provider_display_list
|
||||
from yt_dlp.extractor.youtube.pot._provider import (
|
||||
IEContentProviderLogger,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot.provider import (
|
||||
provider_bug_report_message,
|
||||
)
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import _SkippedComponent
|
||||
from yt_dlp.extractor.youtube.jsc.provider import Preference as JsChallengePreference
|
||||
|
||||
|
||||
class JsChallengeRequestDirector:
|
||||
|
||||
def __init__(self, logger: IEContentProviderLogger):
|
||||
self.providers: dict[str, JsChallengeProvider] = {}
|
||||
self.preferences: list[JsChallengePreference] = []
|
||||
self.logger = logger
|
||||
|
||||
def register_provider(self, provider: JsChallengeProvider):
|
||||
self.providers[provider.PROVIDER_KEY] = provider
|
||||
|
||||
def register_preference(self, preference: JsChallengePreference):
|
||||
self.preferences.append(preference)
|
||||
|
||||
def _get_providers(self, requests: list[JsChallengeRequest]) -> Iterable[JsChallengeProvider]:
|
||||
"""Sorts available providers by preference, given a request"""
|
||||
preferences = {
|
||||
provider: sum(pref(provider, requests) for pref in self.preferences)
|
||||
for provider in self.providers.values()
|
||||
}
|
||||
if self.logger.log_level <= self.logger.LogLevel.TRACE:
|
||||
# calling is_available() for every JS Challenge provider upfront may have some overhead
|
||||
self.logger.trace(f'JS Challenge Providers: {provider_display_list(self.providers.values())}')
|
||||
self.logger.trace('JS Challenge Provider preferences for this request: {}'.format(', '.join(
|
||||
f'{provider.PROVIDER_NAME}={pref}' for provider, pref in preferences.items())))
|
||||
|
||||
return (
|
||||
provider for provider in sorted(
|
||||
self.providers.values(), key=preferences.get, reverse=True)
|
||||
if provider.is_available()
|
||||
)
|
||||
|
||||
def _handle_error(self, e: Exception, provider: JsChallengeProvider, requests: list[JsChallengeRequest]):
|
||||
if isinstance(e, JsChallengeProviderRejectedRequest):
|
||||
self.logger.trace(
|
||||
f'JS Challenge Provider "{provider.PROVIDER_NAME}" rejected '
|
||||
f'{"this request" if len(requests) == 1 else f"{len(requests)} requests"}, '
|
||||
f'trying next available provider. Reason: {e}',
|
||||
)
|
||||
elif isinstance(e, JsChallengeProviderError):
|
||||
if len(requests) == 1:
|
||||
self.logger.warning(
|
||||
f'Error solving {requests[0].type.value} challenge request using "{provider.PROVIDER_NAME}" provider: {e}.\n'
|
||||
f' input = {requests[0].input}\n'
|
||||
f' {(provider_bug_report_message(provider, before="") if not e.expected else "")}')
|
||||
else:
|
||||
self.logger.warning(
|
||||
f'Error solving {len(requests)} challenge requests using "{provider.PROVIDER_NAME}" provider: {e}.\n'
|
||||
f' requests = {requests}\n'
|
||||
f' {(provider_bug_report_message(provider, before="") if not e.expected else "")}')
|
||||
else:
|
||||
self.logger.error(
|
||||
f'Unexpected error solving {len(requests)} challenge request(s) using "{provider.PROVIDER_NAME}" provider: {e!r}\n'
|
||||
f' requests = {requests}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}', cause=e)
|
||||
|
||||
def bulk_solve(self, requests: list[JsChallengeRequest]) -> list[tuple[JsChallengeRequest, JsChallengeResponse]]:
|
||||
"""Solves multiple JS Challenges in bulk, returning a list of responses"""
|
||||
if not self.providers:
|
||||
self.logger.trace('No JS Challenge providers registered')
|
||||
return []
|
||||
|
||||
results = []
|
||||
next_requests = requests[:]
|
||||
|
||||
skipped_components = []
|
||||
for provider in self._get_providers(next_requests):
|
||||
if not next_requests:
|
||||
break
|
||||
self.logger.trace(
|
||||
f'Attempting to solve {len(next_requests)} challenges using "{provider.PROVIDER_NAME}" provider')
|
||||
try:
|
||||
for response in provider.bulk_solve([dataclasses.replace(request) for request in next_requests]):
|
||||
if not validate_provider_response(response):
|
||||
self.logger.warning(
|
||||
f'JS Challenge Provider "{provider.PROVIDER_NAME}" returned an invalid response:'
|
||||
f' response = {response!r}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}')
|
||||
continue
|
||||
if response.error:
|
||||
self._handle_error(response.error, provider, [response.request])
|
||||
continue
|
||||
if (vr_msg := validate_response(response.response, response.request)) is not True:
|
||||
self.logger.warning(
|
||||
f'Invalid JS Challenge response received from "{provider.PROVIDER_NAME}" provider: {vr_msg or ""}\n'
|
||||
f' response = {response.response}\n'
|
||||
f' request = {response.request}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}')
|
||||
continue
|
||||
try:
|
||||
next_requests.remove(response.request)
|
||||
except ValueError:
|
||||
self.logger.warning(
|
||||
f'JS Challenge Provider "{provider.PROVIDER_NAME}" returned a response for an unknown request:\n'
|
||||
f' request = {response.request}\n'
|
||||
f' {provider_bug_report_message(provider, before="")}')
|
||||
continue
|
||||
results.append((response.request, response.response))
|
||||
except Exception as e:
|
||||
if isinstance(e, JsChallengeProviderRejectedRequest) and e._skipped_components:
|
||||
skipped_components.extend(e._skipped_components)
|
||||
self._handle_error(e, provider, next_requests)
|
||||
continue
|
||||
|
||||
if skipped_components:
|
||||
self.__report_skipped_components(skipped_components)
|
||||
|
||||
if len(results) != len(requests):
|
||||
self.logger.trace(
|
||||
f'Not all JS Challenges were solved, expected {len(requests)} responses, got {len(results)}')
|
||||
self.logger.trace(f'Unsolved requests: {next_requests}')
|
||||
else:
|
||||
self.logger.trace(f'Solved all {len(requests)} requested JS Challenges')
|
||||
return results
|
||||
|
||||
def __report_skipped_components(self, components: list[_SkippedComponent], /):
|
||||
runtime_components = collections.defaultdict(list)
|
||||
for component in components:
|
||||
runtime_components[component.component].append(component.runtime)
|
||||
for runtimes in runtime_components.values():
|
||||
runtimes.sort()
|
||||
|
||||
description_lookup = {
|
||||
'ejs:npm': 'NPM package',
|
||||
'ejs:github': 'challenge solver script',
|
||||
}
|
||||
|
||||
descriptions = [
|
||||
f'{description_lookup.get(component, component)} ({", ".join(runtimes)})'
|
||||
for component, runtimes in runtime_components.items()
|
||||
if runtimes
|
||||
]
|
||||
flags = [
|
||||
f' --remote-components {f"{component} (recommended)" if component == "ejs:github" else f"{component} "}'
|
||||
for component, runtimes in runtime_components.items()
|
||||
if runtimes
|
||||
]
|
||||
|
||||
def join_parts(parts, joiner):
|
||||
if not parts:
|
||||
return ''
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
return f'{", ".join(parts[:-1])} {joiner} {parts[-1]}'
|
||||
|
||||
if len(descriptions) > 1:
|
||||
msg = (
|
||||
f'Remote component {descriptions[0]} was skipped. '
|
||||
f'It may be required to solve JS challenges. '
|
||||
f'You can enable the download with {flags[0]}')
|
||||
else:
|
||||
msg = (
|
||||
f'Remote components {join_parts(descriptions, "and")} were skipped. '
|
||||
f'These may be required to solve JS challenges. '
|
||||
f'You can enable these downloads with {join_parts(flags, "or")}, respectively')
|
||||
|
||||
self.logger.warning(f'{msg}. For more information and alternatives, refer to {_EJS_WIKI_URL}')
|
||||
|
||||
def close(self):
|
||||
for provider in self.providers.values():
|
||||
provider.close()
|
||||
|
||||
|
||||
EXTRACTOR_ARG_PREFIX = 'youtubejsc'
|
||||
|
||||
|
||||
def initialize_jsc_director(ie):
|
||||
assert ie._downloader is not None, 'Downloader not set'
|
||||
|
||||
enable_trace = ie._configuration_arg(
|
||||
'jsc_trace', ['false'], ie_key='youtube', casesense=False)[0] == 'true'
|
||||
|
||||
if enable_trace:
|
||||
log_level = IEContentProviderLogger.LogLevel.TRACE
|
||||
elif ie.get_param('verbose', False):
|
||||
log_level = IEContentProviderLogger.LogLevel.DEBUG
|
||||
else:
|
||||
log_level = IEContentProviderLogger.LogLevel.INFO
|
||||
|
||||
def get_provider_logger_and_settings(provider, logger_key):
|
||||
logger_prefix = f'{logger_key}:{provider.PROVIDER_NAME}'
|
||||
extractor_key = f'{EXTRACTOR_ARG_PREFIX}-{provider.PROVIDER_KEY.lower()}'
|
||||
return (
|
||||
YoutubeIEContentProviderLogger(ie, logger_prefix, log_level=log_level),
|
||||
ie.get_param('extractor_args', {}).get(extractor_key, {}))
|
||||
|
||||
director = JsChallengeRequestDirector(
|
||||
logger=YoutubeIEContentProviderLogger(ie, 'jsc', log_level=log_level),
|
||||
)
|
||||
|
||||
ie._downloader.add_close_hook(director.close)
|
||||
|
||||
for provider in _jsc_providers.value.values():
|
||||
logger, settings = get_provider_logger_and_settings(provider, 'jsc')
|
||||
director.register_provider(provider(ie, logger, settings))
|
||||
|
||||
for preference in _jsc_preferences.value:
|
||||
director.register_preference(preference)
|
||||
|
||||
if director.logger.log_level <= director.logger.LogLevel.DEBUG:
|
||||
# calling is_available() for every JS Challenge provider upfront may have some overhead
|
||||
director.logger.debug(f'JS Challenge Providers: {provider_display_list(director.providers.values())}')
|
||||
director.logger.trace(f'Registered {len(director.preferences)} JS Challenge provider preferences')
|
||||
|
||||
return director
|
||||
|
||||
|
||||
def validate_provider_response(response: JsChallengeProviderResponse) -> bool:
|
||||
return (
|
||||
isinstance(response, JsChallengeProviderResponse)
|
||||
and isinstance(response.request, JsChallengeRequest)
|
||||
and (
|
||||
isinstance(response.response, JsChallengeResponse)
|
||||
or (response.error is not None and isinstance(response.error, Exception)))
|
||||
)
|
||||
|
||||
|
||||
def validate_response(response: JsChallengeResponse, request: JsChallengeRequest) -> bool | str:
|
||||
if not isinstance(response, JsChallengeResponse):
|
||||
return 'Response is not a JsChallengeResponse'
|
||||
if request.type == JsChallengeType.N:
|
||||
return validate_nsig_challenge_output(response.output, request.input)
|
||||
else:
|
||||
return validate_sig_challenge_output(response.output, request.input)
|
||||
|
||||
|
||||
def validate_nsig_challenge_output(challenge_output: NChallengeOutput, challenge_input: NChallengeInput) -> bool | str:
|
||||
if not (
|
||||
isinstance(challenge_output, NChallengeOutput)
|
||||
and len(challenge_output.results) == len(challenge_input.challenges)
|
||||
and all(isinstance(k, str) and isinstance(v, str) for k, v in challenge_output.results.items())
|
||||
and all(challenge in challenge_output.results for challenge in challenge_input.challenges)
|
||||
):
|
||||
return 'Invalid NChallengeOutput'
|
||||
|
||||
# Validate n results are valid - if they end with the input challenge then the js function returned with an exception.
|
||||
for challenge, result in challenge_output.results.items():
|
||||
if result.endswith(challenge):
|
||||
return f'n result is invalid for {challenge!r}: {result!r}'
|
||||
return True
|
||||
|
||||
|
||||
def validate_sig_challenge_output(challenge_output: SigChallengeOutput, challenge_input: SigChallengeInput) -> bool:
|
||||
return (
|
||||
isinstance(challenge_output, SigChallengeOutput)
|
||||
and len(challenge_output.results) == len(challenge_input.challenges)
|
||||
and all(isinstance(k, str) and isinstance(v, str) for k, v in challenge_output.results.items())
|
||||
and all(challenge in challenge_output.results for challenge in challenge_input.challenges)
|
||||
) or 'Invalid SigChallengeOutput'
|
||||
@ -0,0 +1,4 @@
|
||||
from yt_dlp.globals import Indirect
|
||||
|
||||
_jsc_providers = Indirect({})
|
||||
_jsc_preferences = Indirect(set())
|
||||
@ -0,0 +1,161 @@
|
||||
"""PUBLIC API"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import dataclasses
|
||||
import enum
|
||||
import typing
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._registry import _jsc_preferences, _jsc_providers
|
||||
from yt_dlp.extractor.youtube.pot._provider import (
|
||||
IEContentProvider,
|
||||
IEContentProviderError,
|
||||
register_preference_generic,
|
||||
register_provider_generic,
|
||||
)
|
||||
from yt_dlp.utils import ExtractorError
|
||||
|
||||
__all__ = [
|
||||
'JsChallengeProvider',
|
||||
'JsChallengeProviderError',
|
||||
'JsChallengeProviderRejectedRequest',
|
||||
'JsChallengeProviderResponse',
|
||||
'JsChallengeRequest',
|
||||
'JsChallengeResponse',
|
||||
'JsChallengeType',
|
||||
'NChallengeInput',
|
||||
'NChallengeOutput',
|
||||
'SigChallengeInput',
|
||||
'SigChallengeOutput',
|
||||
'register_preference',
|
||||
'register_provider',
|
||||
]
|
||||
|
||||
|
||||
class JsChallengeType(enum.Enum):
|
||||
N = 'n'
|
||||
SIG = 'sig'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class JsChallengeRequest:
|
||||
type: JsChallengeType
|
||||
input: NChallengeInput | SigChallengeInput
|
||||
video_id: str | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class NChallengeInput:
|
||||
player_url: str
|
||||
challenges: list[str] = dataclasses.field(default_factory=list)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SigChallengeInput:
|
||||
player_url: str
|
||||
challenges: list[str] = dataclasses.field(default_factory=list)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class NChallengeOutput:
|
||||
results: dict[str, str] = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SigChallengeOutput:
|
||||
results: dict[str, str] = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class JsChallengeProviderResponse:
|
||||
request: JsChallengeRequest
|
||||
response: JsChallengeResponse | None = None
|
||||
error: Exception | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class JsChallengeResponse:
|
||||
type: JsChallengeType
|
||||
output: NChallengeOutput | SigChallengeOutput
|
||||
|
||||
|
||||
class JsChallengeProviderRejectedRequest(IEContentProviderError):
|
||||
"""Reject the JsChallengeRequest (cannot handle the request)"""
|
||||
|
||||
def __init__(self, msg=None, expected: bool = False, *, _skipped_components=None):
|
||||
super().__init__(msg, expected)
|
||||
self._skipped_components = _skipped_components
|
||||
|
||||
|
||||
class JsChallengeProviderError(IEContentProviderError):
|
||||
"""An error occurred while solving the challenge"""
|
||||
|
||||
|
||||
class JsChallengeProvider(IEContentProvider, abc.ABC, suffix='JCP'):
|
||||
|
||||
# Set to None to disable the check
|
||||
_SUPPORTED_TYPES: tuple[JsChallengeType] | None = ()
|
||||
|
||||
def __validate_request(self, request: JsChallengeRequest):
|
||||
if not self.is_available():
|
||||
raise JsChallengeProviderRejectedRequest(f'{self.PROVIDER_NAME} is not available')
|
||||
|
||||
# Validate request using built-in settings
|
||||
if (
|
||||
self._SUPPORTED_TYPES is not None
|
||||
and request.type not in self._SUPPORTED_TYPES
|
||||
):
|
||||
raise JsChallengeProviderRejectedRequest(
|
||||
f'JS Challenge type "{request.type}" is not supported by {self.PROVIDER_NAME}')
|
||||
|
||||
def bulk_solve(self, requests: list[JsChallengeRequest]) -> typing.Generator[JsChallengeProviderResponse, None, None]:
|
||||
"""Solve multiple JS challenges and return the results"""
|
||||
validated_requests = []
|
||||
for request in requests:
|
||||
try:
|
||||
self.__validate_request(request)
|
||||
validated_requests.append(request)
|
||||
except JsChallengeProviderRejectedRequest as e:
|
||||
yield JsChallengeProviderResponse(request=request, error=e)
|
||||
continue
|
||||
yield from self._real_bulk_solve(validated_requests)
|
||||
|
||||
@abc.abstractmethod
|
||||
def _real_bulk_solve(self, requests: list[JsChallengeRequest]) -> typing.Generator[JsChallengeProviderResponse, None, None]:
|
||||
"""Subclasses can override this method to handle bulk solving"""
|
||||
raise NotImplementedError(f'{self.PROVIDER_NAME} does not implement bulk solving')
|
||||
|
||||
def _get_player(self, video_id, player_url):
|
||||
try:
|
||||
return self.ie._load_player(
|
||||
video_id=video_id,
|
||||
player_url=player_url,
|
||||
fatal=True,
|
||||
)
|
||||
except ExtractorError as e:
|
||||
raise JsChallengeProviderError(
|
||||
f'Failed to load player for JS challenge: {e}') from e
|
||||
|
||||
|
||||
def register_provider(provider: type[JsChallengeProvider]):
|
||||
"""Register a JsChallengeProvider class"""
|
||||
return register_provider_generic(
|
||||
provider=provider,
|
||||
base_class=JsChallengeProvider,
|
||||
registry=_jsc_providers.value,
|
||||
)
|
||||
|
||||
|
||||
def register_preference(*providers: type[JsChallengeProvider]) -> typing.Callable[[Preference], Preference]:
|
||||
"""Register a preference for a JsChallengeProvider class."""
|
||||
return register_preference_generic(
|
||||
JsChallengeProvider,
|
||||
_jsc_preferences.value,
|
||||
*providers,
|
||||
)
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
Preference = typing.Callable[[JsChallengeProvider, list[JsChallengeRequest]], int]
|
||||
__all__.append('Preference')
|
||||
@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
import abc
|
||||
import dataclasses
|
||||
import functools
|
||||
|
||||
from ._utils import _get_exe_version_output, detect_exe_version, int_or_none
|
||||
|
||||
|
||||
# NOT public API
|
||||
def runtime_version_tuple(v):
|
||||
# NB: will return (0,) if `v` is an invalid version string
|
||||
return tuple(int_or_none(x, default=0) for x in v.split('.'))
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class JsRuntimeInfo:
|
||||
name: str
|
||||
path: str
|
||||
version: str
|
||||
version_tuple: tuple[int, ...]
|
||||
supported: bool = True
|
||||
|
||||
|
||||
class JsRuntime(abc.ABC):
|
||||
def __init__(self, path=None):
|
||||
self._path = path
|
||||
|
||||
@functools.cached_property
|
||||
def info(self) -> JsRuntimeInfo | None:
|
||||
return self._info()
|
||||
|
||||
@abc.abstractmethod
|
||||
def _info(self) -> JsRuntimeInfo | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DenoJsRuntime(JsRuntime):
|
||||
MIN_SUPPORTED_VERSION = (2, 0, 0)
|
||||
|
||||
def _info(self):
|
||||
path = self._path or 'deno'
|
||||
out = _get_exe_version_output(path, ['--version'])
|
||||
if not out:
|
||||
return None
|
||||
version = detect_exe_version(out, r'^deno (\S+)', 'unknown')
|
||||
vt = runtime_version_tuple(version)
|
||||
return JsRuntimeInfo(
|
||||
name='deno', path=path, version=version, version_tuple=vt,
|
||||
supported=vt >= self.MIN_SUPPORTED_VERSION)
|
||||
|
||||
|
||||
class BunJsRuntime(JsRuntime):
|
||||
MIN_SUPPORTED_VERSION = (1, 0, 31)
|
||||
|
||||
def _info(self):
|
||||
path = self._path or 'bun'
|
||||
out = _get_exe_version_output(path, ['--version'])
|
||||
if not out:
|
||||
return None
|
||||
version = detect_exe_version(out, r'^(\S+)', 'unknown')
|
||||
vt = runtime_version_tuple(version)
|
||||
return JsRuntimeInfo(
|
||||
name='bun', path=path, version=version, version_tuple=vt,
|
||||
supported=vt >= self.MIN_SUPPORTED_VERSION)
|
||||
|
||||
|
||||
class NodeJsRuntime(JsRuntime):
|
||||
MIN_SUPPORTED_VERSION = (20, 0, 0)
|
||||
|
||||
def _info(self):
|
||||
path = self._path or 'node'
|
||||
out = _get_exe_version_output(path, ['--version'])
|
||||
if not out:
|
||||
return None
|
||||
version = detect_exe_version(out, r'^v(\S+)', 'unknown')
|
||||
vt = runtime_version_tuple(version)
|
||||
return JsRuntimeInfo(
|
||||
name='node', path=path, version=version, version_tuple=vt,
|
||||
supported=vt >= self.MIN_SUPPORTED_VERSION)
|
||||
|
||||
|
||||
class QuickJsRuntime(JsRuntime):
|
||||
MIN_SUPPORTED_VERSION = (2023, 12, 9)
|
||||
|
||||
def _info(self):
|
||||
path = self._path or 'qjs'
|
||||
# quickjs does not have --version and --help returns a status code of 1
|
||||
out = _get_exe_version_output(path, ['--help'], ignore_return_code=True)
|
||||
if not out:
|
||||
return None
|
||||
is_ng = 'QuickJS-ng' in out
|
||||
|
||||
version = detect_exe_version(out, r'^QuickJS(?:-ng)?\s+version\s+(\S+)', 'unknown')
|
||||
vt = runtime_version_tuple(version.replace('-', '.'))
|
||||
if is_ng:
|
||||
return JsRuntimeInfo(
|
||||
name='quickjs-ng', path=path, version=version, version_tuple=vt,
|
||||
supported=vt > (0,))
|
||||
return JsRuntimeInfo(
|
||||
name='quickjs', path=path, version=version, version_tuple=vt,
|
||||
supported=vt >= self.MIN_SUPPORTED_VERSION)
|
||||
Loading…
Reference in New Issue