mirror of https://github.com/yt-dlp/yt-dlp
Improve plugin architecture (#5553)
to make plugins easier to develop and use: * Plugins are now loaded as namespace packages. * Plugins can be loaded in any distribution of yt-dlp (binary, pip, source, etc.). * Plugin packages can be installed and managed via pip, or dropped into any of the documented locations. * Users do not need to edit any code files to install plugins. * Backwards-compatible with previous plugin architecture. As a side-effect, yt-dlp will now search in a few more locations for config files. Closes https://github.com/yt-dlp/yt-dlp/issues/1389 Authored by: flashdagger, coletdjnz, pukkandan, Grub4K Co-authored-by: Marcel <flashdagger@googlemail.com> Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com> Co-authored-by: Simon Sawicki <accounts@grub4k.xyz>pull/5921/head
Matthew
2 years ago
committed by
GitHub
parent
2fb0f85868
commit
8e40b9d1ec
@ -0,0 +1,73 @@
|
||||
import importlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
TEST_DATA_DIR = Path(os.path.dirname(os.path.abspath(__file__)), 'testdata')
|
||||
sys.path.append(str(TEST_DATA_DIR))
|
||||
importlib.invalidate_caches()
|
||||
|
||||
from yt_dlp.plugins import PACKAGE_NAME, directories, load_plugins
|
||||
|
||||
|
||||
class TestPlugins(unittest.TestCase):
|
||||
|
||||
TEST_PLUGIN_DIR = TEST_DATA_DIR / PACKAGE_NAME
|
||||
|
||||
def test_directories_containing_plugins(self):
|
||||
self.assertIn(self.TEST_PLUGIN_DIR, map(Path, directories()))
|
||||
|
||||
def test_extractor_classes(self):
|
||||
for module_name in tuple(sys.modules):
|
||||
if module_name.startswith(f'{PACKAGE_NAME}.extractor'):
|
||||
del sys.modules[module_name]
|
||||
plugins_ie = load_plugins('extractor', 'IE')
|
||||
|
||||
self.assertIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys())
|
||||
self.assertIn('NormalPluginIE', plugins_ie.keys())
|
||||
|
||||
# don't load modules with underscore prefix
|
||||
self.assertFalse(
|
||||
f'{PACKAGE_NAME}.extractor._ignore' in sys.modules.keys(),
|
||||
'loaded module beginning with underscore')
|
||||
self.assertNotIn('IgnorePluginIE', plugins_ie.keys())
|
||||
|
||||
# Don't load extractors with underscore prefix
|
||||
self.assertNotIn('_IgnoreUnderscorePluginIE', plugins_ie.keys())
|
||||
|
||||
# Don't load extractors not specified in __all__ (if supplied)
|
||||
self.assertNotIn('IgnoreNotInAllPluginIE', plugins_ie.keys())
|
||||
self.assertIn('InAllPluginIE', plugins_ie.keys())
|
||||
|
||||
def test_postprocessor_classes(self):
|
||||
plugins_pp = load_plugins('postprocessor', 'PP')
|
||||
self.assertIn('NormalPluginPP', plugins_pp.keys())
|
||||
|
||||
def test_importing_zipped_module(self):
|
||||
zip_path = TEST_DATA_DIR / 'zipped_plugins.zip'
|
||||
shutil.make_archive(str(zip_path)[:-4], 'zip', str(zip_path)[:-4])
|
||||
sys.path.append(str(zip_path)) # add zip to search paths
|
||||
importlib.invalidate_caches() # reset the import caches
|
||||
|
||||
try:
|
||||
for plugin_type in ('extractor', 'postprocessor'):
|
||||
package = importlib.import_module(f'{PACKAGE_NAME}.{plugin_type}')
|
||||
self.assertIn(zip_path / PACKAGE_NAME / plugin_type, map(Path, package.__path__))
|
||||
|
||||
plugins_ie = load_plugins('extractor', 'IE')
|
||||
self.assertIn('ZippedPluginIE', plugins_ie.keys())
|
||||
|
||||
plugins_pp = load_plugins('postprocessor', 'PP')
|
||||
self.assertIn('ZippedPluginPP', plugins_pp.keys())
|
||||
|
||||
finally:
|
||||
sys.path.remove(str(zip_path))
|
||||
os.remove(zip_path)
|
||||
importlib.invalidate_caches() # reset the import caches
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,5 @@
|
||||
from yt_dlp.extractor.common import InfoExtractor
|
||||
|
||||
|
||||
class IgnorePluginIE(InfoExtractor):
|
||||
pass
|
||||
@ -0,0 +1,12 @@
|
||||
from yt_dlp.extractor.common import InfoExtractor
|
||||
|
||||
|
||||
class IgnoreNotInAllPluginIE(InfoExtractor):
|
||||
pass
|
||||
|
||||
|
||||
class InAllPluginIE(InfoExtractor):
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ['InAllPluginIE']
|
||||
@ -0,0 +1,9 @@
|
||||
from yt_dlp.extractor.common import InfoExtractor
|
||||
|
||||
|
||||
class NormalPluginIE(InfoExtractor):
|
||||
pass
|
||||
|
||||
|
||||
class _IgnoreUnderscorePluginIE(InfoExtractor):
|
||||
pass
|
||||
@ -0,0 +1,5 @@
|
||||
from yt_dlp.postprocessor.common import PostProcessor
|
||||
|
||||
|
||||
class NormalPluginPP(PostProcessor):
|
||||
pass
|
||||
@ -0,0 +1,5 @@
|
||||
from yt_dlp.extractor.common import InfoExtractor
|
||||
|
||||
|
||||
class ZippedPluginIE(InfoExtractor):
|
||||
pass
|
||||
@ -0,0 +1,5 @@
|
||||
from yt_dlp.postprocessor.common import PostProcessor
|
||||
|
||||
|
||||
class ZippedPluginPP(PostProcessor):
|
||||
pass
|
||||
@ -0,0 +1,171 @@
|
||||
import contextlib
|
||||
import importlib
|
||||
import importlib.abc
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import inspect
|
||||
import itertools
|
||||
import os
|
||||
import pkgutil
|
||||
import sys
|
||||
import traceback
|
||||
import zipimport
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
from .compat import functools # isort: split
|
||||
from .compat import compat_expanduser
|
||||
from .utils import (
|
||||
get_executable_path,
|
||||
get_system_config_dirs,
|
||||
get_user_config_dirs,
|
||||
write_string,
|
||||
)
|
||||
|
||||
PACKAGE_NAME = 'yt_dlp_plugins'
|
||||
COMPAT_PACKAGE_NAME = 'ytdlp_plugins'
|
||||
|
||||
|
||||
class PluginLoader(importlib.abc.Loader):
|
||||
"""Dummy loader for virtual namespace packages"""
|
||||
|
||||
def exec_module(self, module):
|
||||
return None
|
||||
|
||||
|
||||
@functools.cache
|
||||
def dirs_in_zip(archive):
|
||||
with ZipFile(archive) as zip:
|
||||
return set(itertools.chain.from_iterable(
|
||||
Path(file).parents for file in zip.namelist()))
|
||||
|
||||
|
||||
class PluginFinder(importlib.abc.MetaPathFinder):
|
||||
"""
|
||||
This class provides one or multiple namespace packages.
|
||||
It searches in sys.path and yt-dlp config folders for
|
||||
the existing subdirectories from which the modules can be imported
|
||||
"""
|
||||
|
||||
def __init__(self, *packages):
|
||||
self._zip_content_cache = {}
|
||||
self.packages = set(itertools.chain.from_iterable(
|
||||
itertools.accumulate(name.split('.'), lambda a, b: '.'.join((a, b)))
|
||||
for name in packages))
|
||||
|
||||
def search_locations(self, fullname):
|
||||
candidate_locations = []
|
||||
|
||||
def _get_package_paths(*root_paths, containing_folder='plugins'):
|
||||
for config_dir in map(Path, root_paths):
|
||||
plugin_dir = config_dir / containing_folder
|
||||
if not plugin_dir.is_dir():
|
||||
continue
|
||||
yield from plugin_dir.iterdir()
|
||||
|
||||
# Load from yt-dlp config folders
|
||||
candidate_locations.extend(_get_package_paths(
|
||||
*get_user_config_dirs('yt-dlp'), *get_system_config_dirs('yt-dlp'),
|
||||
containing_folder='plugins'))
|
||||
|
||||
# Load from yt-dlp-plugins folders
|
||||
candidate_locations.extend(_get_package_paths(
|
||||
get_executable_path(),
|
||||
compat_expanduser('~'),
|
||||
'/etc',
|
||||
os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config'),
|
||||
containing_folder='yt-dlp-plugins'))
|
||||
|
||||
candidate_locations.extend(map(Path, sys.path)) # PYTHONPATH
|
||||
|
||||
parts = Path(*fullname.split('.'))
|
||||
locations = set()
|
||||
for path in dict.fromkeys(candidate_locations):
|
||||
candidate = path / parts
|
||||
if candidate.is_dir():
|
||||
locations.add(str(candidate))
|
||||
elif path.name and any(path.with_suffix(suffix).is_file() for suffix in {'.zip', '.egg', '.whl'}):
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
if parts in dirs_in_zip(path):
|
||||
locations.add(str(candidate))
|
||||
return locations
|
||||
|
||||
def find_spec(self, fullname, path=None, target=None):
|
||||
if fullname not in self.packages:
|
||||
return None
|
||||
|
||||
search_locations = self.search_locations(fullname)
|
||||
if not search_locations:
|
||||
return None
|
||||
|
||||
spec = importlib.machinery.ModuleSpec(fullname, PluginLoader(), is_package=True)
|
||||
spec.submodule_search_locations = search_locations
|
||||
return spec
|
||||
|
||||
def invalidate_caches(self):
|
||||
dirs_in_zip.cache_clear()
|
||||
for package in self.packages:
|
||||
if package in sys.modules:
|
||||
del sys.modules[package]
|
||||
|
||||
|
||||
def directories():
|
||||
spec = importlib.util.find_spec(PACKAGE_NAME)
|
||||
return spec.submodule_search_locations if spec else []
|
||||
|
||||
|
||||
def iter_modules(subpackage):
|
||||
fullname = f'{PACKAGE_NAME}.{subpackage}'
|
||||
with contextlib.suppress(ModuleNotFoundError):
|
||||
pkg = importlib.import_module(fullname)
|
||||
yield from pkgutil.iter_modules(path=pkg.__path__, prefix=f'{fullname}.')
|
||||
|
||||
|
||||
def load_module(module, module_name, suffix):
|
||||
return inspect.getmembers(module, lambda obj: (
|
||||
inspect.isclass(obj)
|
||||
and obj.__name__.endswith(suffix)
|
||||
and obj.__module__.startswith(module_name)
|
||||
and not obj.__name__.startswith('_')
|
||||
and obj.__name__ in getattr(module, '__all__', [obj.__name__])))
|
||||
|
||||
|
||||
def load_plugins(name, suffix):
|
||||
classes = {}
|
||||
|
||||
for finder, module_name, _ in iter_modules(name):
|
||||
if any(x.startswith('_') for x in module_name.split('.')):
|
||||
continue
|
||||
try:
|
||||
if sys.version_info < (3, 10) and isinstance(finder, zipimport.zipimporter):
|
||||
# zipimporter.load_module() is deprecated in 3.10 and removed in 3.12
|
||||
# The exec_module branch below is the replacement for >= 3.10
|
||||
# See: https://docs.python.org/3/library/zipimport.html#zipimport.zipimporter.exec_module
|
||||
module = finder.load_module(module_name)
|
||||
else:
|
||||
spec = finder.find_spec(module_name)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = module
|
||||
spec.loader.exec_module(module)
|
||||
except Exception:
|
||||
write_string(f'Error while importing module {module_name!r}\n{traceback.format_exc(limit=-1)}')
|
||||
continue
|
||||
classes.update(load_module(module, module_name, suffix))
|
||||
|
||||
# Compat: old plugin system using __init__.py
|
||||
# Note: plugins imported this way do not show up in directories()
|
||||
# nor are considered part of the yt_dlp_plugins namespace package
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
name, Path(get_executable_path(), COMPAT_PACKAGE_NAME, name, '__init__.py'))
|
||||
plugins = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = plugins
|
||||
spec.loader.exec_module(plugins)
|
||||
classes.update(load_module(plugins, spec.name, suffix))
|
||||
|
||||
return classes
|
||||
|
||||
|
||||
sys.meta_path.insert(0, PluginFinder(f'{PACKAGE_NAME}.extractor', f'{PACKAGE_NAME}.postprocessor'))
|
||||
|
||||
__all__ = ['directories', 'load_plugins', 'PACKAGE_NAME', 'COMPAT_PACKAGE_NAME']
|
||||
Loading…
Reference in New Issue