[plugins] Load plugins on demand

README.md

@@ -338,10 +338,11 @@ If you fork the project on GitHub, you can run your fork's [build workflow](.git
     --plugin-dirs PATH              Path to an additional directory to search
                                     for plugins. This option can be used
                                     multiple times to add multiple directories.
-                                    Note that this currently only works for
-                                    extractor plugins; postprocessor plugins can
-                                    only be loaded from the default plugin
-                                    directories
+                                    Use "default" to search the default plugin
+                                    directories (default)
+    --no-plugin-dirs                Clear plugin directories to search,
+                                    including defaults and those provided by
+                                    previous --plugin-dirs
     --flat-playlist                 Do not extract a playlist's URL result
                                     entries; some entry metadata may be missing
                                     and downloading may be bypassed

devscripts/make_lazy_extractors.py

@@ -10,6 +10,9 @@
 from inspect import getsource
 
 from devscripts.utils import get_filename_args, read_file, write_file
+from yt_dlp.extractor import import_extractors
+from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor
+from yt_dlp.globals import extractors
 
 NO_ATTR = object()
 STATIC_CLASS_PROPERTIES = [
@@ -38,16 +41,15 @@ def main():
 
     lazy_extractors_filename = get_filename_args(default_outfile='yt_dlp/extractor/lazy_extractors.py')
 
-    from yt_dlp.extractor.extractors import _ALL_CLASSES
-    from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor
+    import_extractors()
 
     DummyInfoExtractor = type('InfoExtractor', (InfoExtractor,), {'IE_NAME': NO_ATTR})
     module_src = '\n'.join((
         MODULE_TEMPLATE,
         '    _module = None',
         *extra_ie_code(DummyInfoExtractor),
         '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n',
-        *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor),
+        *build_ies(list(extractors.value.values()), (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor),
     ))
 
     write_file(lazy_extractors_filename, f'{module_src}\n')
@@ -73,7 +75,8 @@ def build_ies(ies, bases, attr_base):
         if ie in ies:
             names.append(ie.__name__)
 
-    yield f'\n_ALL_CLASSES = [{", ".join(names)}]'
+    class_lookup_contents = ', '.join(f'{name!r}: {name}' for name in names)
+    yield f'\n_CLASS_LOOKUP = {{ {class_lookup_contents} }}'
 
 
 def sort_ies(ies, ignored_bases):

test/test_YoutubeDL.py

@@ -6,6 +6,8 @@
 import unittest
 from unittest.mock import patch
 
+from yt_dlp.globals import all_plugins_loaded
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
@@ -1427,6 +1429,12 @@ def check_for_cookie_header(result):
         self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain')
         self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain')
 
+    def test_load_plugins_compat(self):
+        # Should try to reload plugins if they haven't already been loaded
+        all_plugins_loaded.value = False
+        FakeYDL().close()
+        assert all_plugins_loaded.value
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_plugins.py

@@ -10,22 +10,74 @@
 sys.path.append(str(TEST_DATA_DIR))
 importlib.invalidate_caches()
 
-from yt_dlp.utils import Config
-from yt_dlp.plugins import PACKAGE_NAME, directories, load_plugins
+from yt_dlp.plugins import (
+    PACKAGE_NAME,
+    PluginSpec,
+    directories,
+    load_plugins,
+    load_all_plugins,
+    register_plugin_spec,
+    clear_plugins,
+)
+
+from yt_dlp.globals import (
+    extractors,
+    postprocessors,
+    plugin_dirs,
+    plugin_ies,
+    plugin_pps,
+    all_plugins_loaded,
+    plugin_specs,
+)
+
+from yt_dlp.utils import YoutubeDLError
+
+
+EXTRACTOR_PLUGIN_SPEC = PluginSpec(
+    module_name='extractor',
+    suffix='IE',
+    destination=extractors,
+    plugin_destination=plugin_ies,
+)
+
+POSTPROCESSOR_PLUGIN_SPEC = PluginSpec(
+    module_name='postprocessor',
+    suffix='PP',
+    destination=postprocessors,
+    plugin_destination=plugin_pps,
+)
+
+
+def reset_plugins():
+    plugin_ies.value = {}
+    plugin_pps.value = {}
+    plugin_dirs.value = ['default']
+    plugin_specs.value = {}
+    all_plugins_loaded.value = False
+    # Clearing override plugins is probably difficult
+    for module_name in tuple(sys.modules):
+        for plugin_type in ('extractor', 'postprocessor'):
+            if module_name.startswith(f'{PACKAGE_NAME}.{plugin_type}.'):
+                del sys.modules[module_name]
+
+    importlib.invalidate_caches()
 
 
 class TestPlugins(unittest.TestCase):
 
     TEST_PLUGIN_DIR = TEST_DATA_DIR / PACKAGE_NAME
 
+    def setUp(self):
+        reset_plugins()
+
+    def tearDown(self):
+        reset_plugins()
+
     def test_directories_containing_plugins(self):
         self.assertIn(self.TEST_PLUGIN_DIR, map(Path, directories()))
 
     def test_extractor_classes(self):
-        for module_name in tuple(sys.modules):
-            if module_name.startswith(f'{PACKAGE_NAME}.extractor'):
-                del sys.modules[module_name]
-        plugins_ie = load_plugins('extractor', 'IE')
+        plugins_ie = load_plugins(EXTRACTOR_PLUGIN_SPEC)
 
         self.assertIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys())
         self.assertIn('NormalPluginIE', plugins_ie.keys())
@@ -35,17 +87,29 @@ def test_extractor_classes(self):
             f'{PACKAGE_NAME}.extractor._ignore' in sys.modules,
             'loaded module beginning with underscore')
         self.assertNotIn('IgnorePluginIE', plugins_ie.keys())
+        self.assertNotIn('IgnorePluginIE', plugin_ies.value)
 
         # Don't load extractors with underscore prefix
         self.assertNotIn('_IgnoreUnderscorePluginIE', plugins_ie.keys())
+        self.assertNotIn('_IgnoreUnderscorePluginIE', plugin_ies.value)
 
         # Don't load extractors not specified in __all__ (if supplied)
         self.assertNotIn('IgnoreNotInAllPluginIE', plugins_ie.keys())
+        self.assertNotIn('IgnoreNotInAllPluginIE', plugin_ies.value)
         self.assertIn('InAllPluginIE', plugins_ie.keys())
+        self.assertIn('InAllPluginIE', plugin_ies.value)
+
+        # Don't load override extractors
+        self.assertNotIn('OverrideGenericIE', plugins_ie.keys())
+        self.assertNotIn('OverrideGenericIE', plugin_ies.value)
+        self.assertNotIn('_UnderscoreOverrideGenericIE', plugins_ie.keys())
+        self.assertNotIn('_UnderscoreOverrideGenericIE', plugin_ies.value)
 
     def test_postprocessor_classes(self):
-        plugins_pp = load_plugins('postprocessor', 'PP')
+        plugins_pp = load_plugins(POSTPROCESSOR_PLUGIN_SPEC)
         self.assertIn('NormalPluginPP', plugins_pp.keys())
+        self.assertIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys())
+        self.assertIn('NormalPluginPP', plugin_pps.value)
 
     def test_importing_zipped_module(self):
         zip_path = TEST_DATA_DIR / 'zipped_plugins.zip'
@@ -58,34 +122,150 @@ def test_importing_zipped_module(self):
                 package = importlib.import_module(f'{PACKAGE_NAME}.{plugin_type}')
                 self.assertIn(zip_path / PACKAGE_NAME / plugin_type, map(Path, package.__path__))
 
-            plugins_ie = load_plugins('extractor', 'IE')
+            plugins_ie = load_plugins(EXTRACTOR_PLUGIN_SPEC)
             self.assertIn('ZippedPluginIE', plugins_ie.keys())
 
-            plugins_pp = load_plugins('postprocessor', 'PP')
+            plugins_pp = load_plugins(POSTPROCESSOR_PLUGIN_SPEC)
             self.assertIn('ZippedPluginPP', plugins_pp.keys())
 
         finally:
             sys.path.remove(str(zip_path))
             os.remove(zip_path)
             importlib.invalidate_caches()  # reset the import caches
 
-    def test_plugin_dirs(self):
-        # Internal plugin dirs hack for CLI --plugin-dirs
-        # To be replaced with proper system later
-        custom_plugin_dir = TEST_DATA_DIR / 'plugin_packages'
-        Config._plugin_dirs = [str(custom_plugin_dir)]
-        importlib.invalidate_caches()  # reset the import caches
+    def test_reloading_plugins(self):
+        reload_plugins_path = TEST_DATA_DIR / 'reload_plugins'
+        load_plugins(EXTRACTOR_PLUGIN_SPEC)
+        load_plugins(POSTPROCESSOR_PLUGIN_SPEC)
 
+        # Remove default folder and add reload_plugin path
+        sys.path.remove(str(TEST_DATA_DIR))
+        sys.path.append(str(reload_plugins_path))
+        importlib.invalidate_caches()
         try:
-            package = importlib.import_module(f'{PACKAGE_NAME}.extractor')
-            self.assertIn(custom_plugin_dir / 'testpackage' / PACKAGE_NAME / 'extractor', map(Path, package.__path__))
+            for plugin_type in ('extractor', 'postprocessor'):
+                package = importlib.import_module(f'{PACKAGE_NAME}.{plugin_type}')
+                self.assertIn(reload_plugins_path / PACKAGE_NAME / plugin_type, map(Path, package.__path__))
+
+            plugins_ie = load_plugins(EXTRACTOR_PLUGIN_SPEC)
+            self.assertIn('NormalPluginIE', plugins_ie.keys())
+            self.assertTrue(
+                plugins_ie['NormalPluginIE'].REPLACED,
+                msg='Reloading has not replaced original extractor plugin')
+            self.assertTrue(
+                extractors.value['NormalPluginIE'].REPLACED,
+                msg='Reloading has not replaced original extractor plugin globally')
 
-            plugins_ie = load_plugins('extractor', 'IE')
-            self.assertIn('PackagePluginIE', plugins_ie.keys())
+            plugins_pp = load_plugins(POSTPROCESSOR_PLUGIN_SPEC)
+            self.assertIn('NormalPluginPP', plugins_pp.keys())
+            self.assertTrue(plugins_pp['NormalPluginPP'].REPLACED,
+                            msg='Reloading has not replaced original postprocessor plugin')
+            self.assertTrue(
+                postprocessors.value['NormalPluginPP'].REPLACED,
+                msg='Reloading has not replaced original postprocessor plugin globally')
 
         finally:
-            Config._plugin_dirs = []
-            importlib.invalidate_caches()  # reset the import caches
+            sys.path.remove(str(reload_plugins_path))
+            sys.path.append(str(TEST_DATA_DIR))
+            importlib.invalidate_caches()
+
+    def test_extractor_override_plugin(self):
+        load_plugins(EXTRACTOR_PLUGIN_SPEC)
+
+        from yt_dlp.extractor.generic import GenericIE
+
+        self.assertEqual(GenericIE.TEST_FIELD, 'override')
+        self.assertEqual(GenericIE.SECONDARY_TEST_FIELD, 'underscore-override')
+
+        self.assertEqual(GenericIE.IE_NAME, 'generic+override+underscore-override')
+        importlib.invalidate_caches()
+        #  test that loading a second time doesn't wrap a second time
+        load_plugins(EXTRACTOR_PLUGIN_SPEC)
+        from yt_dlp.extractor.generic import GenericIE
+        self.assertEqual(GenericIE.IE_NAME, 'generic+override+underscore-override')
+
+    def test_load_all_plugin_types(self):
+
+        # no plugin specs registered
+        load_all_plugins()
+
+        self.assertNotIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys())
+        self.assertNotIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys())
+
+        register_plugin_spec(EXTRACTOR_PLUGIN_SPEC)
+        register_plugin_spec(POSTPROCESSOR_PLUGIN_SPEC)
+        load_all_plugins()
+        self.assertTrue(all_plugins_loaded.value)
+
+        self.assertIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys())
+        self.assertIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys())
+
+    def test_no_plugin_dirs(self):
+        register_plugin_spec(EXTRACTOR_PLUGIN_SPEC)
+        register_plugin_spec(POSTPROCESSOR_PLUGIN_SPEC)
+
+        plugin_dirs.value = []
+        load_all_plugins()
+
+        self.assertNotIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys())
+        self.assertNotIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys())
+
+    def test_set_plugin_dirs(self):
+        custom_plugin_dir = str(TEST_DATA_DIR / 'plugin_packages')
+        plugin_dirs.value = [custom_plugin_dir]
+
+        load_plugins(EXTRACTOR_PLUGIN_SPEC)
+
+        self.assertIn(f'{PACKAGE_NAME}.extractor.package', sys.modules.keys())
+        self.assertIn('PackagePluginIE', plugin_ies.value)
+
+    def test_invalid_plugin_dir(self):
+        plugin_dirs.value = ['invalid_dir']
+        with self.assertRaises(ValueError):
+            load_plugins(EXTRACTOR_PLUGIN_SPEC)
+
+    def test_append_plugin_dirs(self):
+        custom_plugin_dir = str(TEST_DATA_DIR / 'plugin_packages')
+
+        self.assertEqual(plugin_dirs.value, ['default'])
+        plugin_dirs.value.append(custom_plugin_dir)
+        self.assertEqual(plugin_dirs.value, ['default', custom_plugin_dir])
+
+        load_plugins(EXTRACTOR_PLUGIN_SPEC)
+
+        self.assertIn(f'{PACKAGE_NAME}.extractor.package', sys.modules.keys())
+        self.assertIn('PackagePluginIE', plugin_ies.value)
+
+    def test_clear_plugins(self):
+        clear_plugins()
+        ies = load_plugins(EXTRACTOR_PLUGIN_SPEC)
+        self.assertEqual(ies, {})
+        self.assertNotIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys())
+        self.assertNotIn('NormalPluginIE', plugin_ies.value)
+
+        pps = load_plugins(POSTPROCESSOR_PLUGIN_SPEC)
+        self.assertEqual(pps, {})
+        self.assertNotIn(f'{PACKAGE_NAME}.postprocessor.normal', sys.modules.keys())
+        self.assertNotIn('NormalPluginPP', plugin_pps.value)
+
+    def test_clear_plugins_already_loaded(self):
+        register_plugin_spec(EXTRACTOR_PLUGIN_SPEC)
+        register_plugin_spec(POSTPROCESSOR_PLUGIN_SPEC)
+        load_all_plugins()
+
+        with self.assertRaises(YoutubeDLError):
+            clear_plugins()
+
+        ies = load_plugins(EXTRACTOR_PLUGIN_SPEC)
+        self.assertIn('NormalPluginIE', ies)
+
+    def test_get_plugin_spec(self):
+        register_plugin_spec(EXTRACTOR_PLUGIN_SPEC)
+        register_plugin_spec(POSTPROCESSOR_PLUGIN_SPEC)
+
+        self.assertEqual(plugin_specs.value.get('extractor'), EXTRACTOR_PLUGIN_SPEC)
+        self.assertEqual(plugin_specs.value.get('postprocessor'), POSTPROCESSOR_PLUGIN_SPEC)
+        self.assertIsNone(plugin_specs.value.get('invalid'))
 
 
 if __name__ == '__main__':

test/testdata/plugin_packages/testpackage/yt_dlp_plugins/extractor/package.py

@@ -2,4 +2,5 @@
 
 
 class PackagePluginIE(InfoExtractor):
+    _VALID_URL = 'package'
     pass

test/testdata/reload_plugins/yt_dlp_plugins/extractor/normal.py

@@ -0,0 +1,10 @@
+from yt_dlp.extractor.common import InfoExtractor
+
+
+class NormalPluginIE(InfoExtractor):
+    _VALID_URL = 'normal'
+    REPLACED = True
+
+
+class _IgnoreUnderscorePluginIE(InfoExtractor):
+    pass

test/testdata/reload_plugins/yt_dlp_plugins/postprocessor/normal.py

@@ -0,0 +1,5 @@
+from yt_dlp.postprocessor.common import PostProcessor
+
+
+class NormalPluginPP(PostProcessor):
+    REPLACED = True

test/testdata/yt_dlp_plugins/extractor/ignore.py

@@ -6,6 +6,7 @@ class IgnoreNotInAllPluginIE(InfoExtractor):
 
 
 class InAllPluginIE(InfoExtractor):
+    _VALID_URL = 'inallpluginie'
     pass
 
 

test/testdata/yt_dlp_plugins/extractor/normal.py

@@ -2,8 +2,10 @@
 
 
 class NormalPluginIE(InfoExtractor):
-    pass
+    _VALID_URL = 'normalpluginie'
+    REPLACED = False
 
 
 class _IgnoreUnderscorePluginIE(InfoExtractor):
+    _VALID_URL = 'ignoreunderscorepluginie'
     pass