[youtube] Improvements to JS player extraction (See desc) (#860)

* fallback player url extraction when it fails to be extracted from the webpage
* don't download js player unnecessarily for clients that don't require it
* try to extract js player url from any additional client configs
* ability to skip the js player usage/download using `player_skip=js`
* ability to skip the initial webpage download using `player_skip=webpage`

known issue:
* authentication for multi-channel accounts and multi-account cookies may not work correctly if the webpage or client configs are skipped
*  formats from the web client requiring signature decryption will be skipped if player js extraction is skipped

Authored by: coletdjnz

README.md

@@ -1436,7 +1436,7 @@ The following extractors use this feature:
 * **youtube**
     * `skip`: `hls` or `dash` (or both) to skip download of the respective manifests
     * `player_client`: Clients to extract video data from. The main clients are `web`, `android`, `ios`, `mweb`. These also have `_music`, `_embedded`, `_agegate`, and `_creator` variants (Eg: `web_embedded`) (`mweb` has only `_agegate`). By default, `android,web` is used, but the agegate and creator variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can also use `all` to use all the clients
-    * `player_skip`: `configs` - skip any requests for client configs and use defaults
+    * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
     * `include_live_dash`: Include live dash formats (These formats don't download properly)
     * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side).
     * `max_comments`: Maximum amount of comments to download (default all).

yt_dlp/extractor/youtube.py

@@ -117,6 +117,7 @@ INNERTUBE_CLIENTS = {
             }
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
+        'REQUIRE_JS_PLAYER': False
     },
     'android_embedded': {
         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
@@ -126,7 +127,8 @@ INNERTUBE_CLIENTS = {
                 'clientVersion': '16.20',
             },
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 55
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
+        'REQUIRE_JS_PLAYER': False
     },
     'android_music': {
         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
@@ -138,6 +140,7 @@ INNERTUBE_CLIENTS = {
             }
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
+        'REQUIRE_JS_PLAYER': False
     },
     'android_creator': {
         'INNERTUBE_CONTEXT': {
@@ -146,7 +149,8 @@ INNERTUBE_CLIENTS = {
                 'clientVersion': '21.24.100',
             },
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 14
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
+        'REQUIRE_JS_PLAYER': False
     },
     # ios has HLS live streams
     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680
@@ -158,7 +162,8 @@ INNERTUBE_CLIENTS = {
                 'clientVersion': '16.20',
             }
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 5
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
+        'REQUIRE_JS_PLAYER': False
     },
     'ios_embedded': {
         'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
@@ -168,7 +173,8 @@ INNERTUBE_CLIENTS = {
                 'clientVersion': '16.20',
             },
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 66
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
+        'REQUIRE_JS_PLAYER': False
     },
     'ios_music': {
         'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
@@ -179,7 +185,8 @@ INNERTUBE_CLIENTS = {
                 'clientVersion': '4.32',
             },
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 26
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
+        'REQUIRE_JS_PLAYER': False
     },
     'ios_creator': {
         'INNERTUBE_CONTEXT': {
@@ -188,7 +195,8 @@ INNERTUBE_CLIENTS = {
                 'clientVersion': '21.24.100',
             },
         },
-        'INNERTUBE_CONTEXT_CLIENT_NAME': 15
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
+        'REQUIRE_JS_PLAYER': False
     },
     # mweb has 'ultralow' formats
     # See: https://github.com/yt-dlp/yt-dlp/pull/557
@@ -215,6 +223,7 @@ def build_innertube_clients():
     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
+        ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
         ytcfg['priority'] = 10 * priority(client.split('_', 1)[0])
 
@@ -1858,14 +1867,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         self._code_cache = {}
         self._player_cache = {}
 
-    def _extract_player_url(self, ytcfg=None, webpage=None):
-        player_url = try_get(ytcfg, (lambda x: x['PLAYER_JS_URL']), str)
-        if not player_url and webpage:
-            player_url = self._search_regex(
-                r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
-                webpage, 'player URL', fatal=False)
+    def _extract_player_url(self, *ytcfgs, webpage=None):
+        player_url = traverse_obj(
+            ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
+            get_all=False, expected_type=compat_str)
         if not player_url:
-            return None
+            return
         if player_url.startswith('//'):
             player_url = 'https:' + player_url
         elif not re.match(r'https?://', player_url):
@@ -1873,6 +1880,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'https://www.youtube.com', player_url)
         return player_url
 
+    def _download_player_url(self, video_id, fatal=False):
+        res = self._download_webpage(
+            'https://www.youtube.com/iframe_api',
+            note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
+        if res:
+            player_version = self._search_regex(
+                r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
+            if player_version:
+                return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
+
     def _signature_cache_id(self, example_sig):
         """ Return a string representation of a signature """
         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
@@ -2462,7 +2479,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
-        sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False)
+        sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
         headers = self.generate_api_headers(
             player_ytcfg, identity_token, syncid,
             default_client=client, session_index=session_index)
@@ -2507,7 +2524,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         webpage = self._download_webpage(url, video_id, fatal=False, note=f'Downloading {client} config')
         return self.extract_ytcfg(video_id, webpage) or {}
 
-    def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, player_url, identity_token):
+    def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, identity_token):
         initial_pr = None
         if webpage:
             initial_pr = self._extract_yt_initial_variable(
@@ -2516,6 +2533,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         original_clients = clients
         clients = clients[::-1]
+        prs = []
 
         def append_client(client_name):
             if client_name in INNERTUBE_CLIENTS and client_name not in original_clients:
@@ -2525,23 +2543,33 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # extraction of some data. So we return the initial_pr with formats
         # stripped out even if not requested by the user
         # See: https://github.com/yt-dlp/yt-dlp/issues/501
-        yielded_pr = False
         if initial_pr:
             pr = dict(initial_pr)
             pr['streamingData'] = None
-            yielded_pr = True
-            yield pr
+            prs.append(pr)
 
         last_error = None
+        tried_iframe_fallback = False
+        player_url = None
         while clients:
             client = clients.pop()
             player_ytcfg = master_ytcfg if client == 'web' else {}
             if 'configs' not in self._configuration_arg('player_skip'):
                 player_ytcfg = self._extract_player_ytcfg(client, video_id) or player_ytcfg
 
+            player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
+            require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
+            if 'js' in self._configuration_arg('player_skip'):
+                require_js_player = False
+                player_url = None
+
+            if not player_url and not tried_iframe_fallback and require_js_player:
+                player_url = self._download_player_url(video_id)
+                tried_iframe_fallback = True
+
             try:
                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
-                    client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, identity_token, player_url, initial_pr)
+                    client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, identity_token, player_url if require_js_player else None, initial_pr)
             except ExtractorError as e:
                 if last_error:
                     self.report_warning(last_error)
@@ -2549,8 +2577,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 continue
 
             if pr:
-                yielded_pr = True
-                yield pr
+                prs.append(pr)
 
             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
             if client.endswith('_agegate') and self._is_unplayable(pr) and self._generate_sapisidhash_header():
@@ -2559,9 +2586,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 append_client(f'{client}_agegate')
 
         if last_error:
-            if not yielded_pr:
+            if not len(prs):
                 raise last_error
             self.report_warning(last_error)
+        return prs, player_url
 
     def _extract_formats(self, streaming_data, video_id, player_url, is_live):
         itags, stream_ids = [], []
@@ -2708,16 +2736,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         base_url = self.http_scheme() + '//www.youtube.com/'
         webpage_url = base_url + 'watch?v=' + video_id
+        webpage = None
+        if 'webpage' not in self._configuration_arg('player_skip'):
             webpage = self._download_webpage(
                 webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
 
         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
-        player_url = self._extract_player_url(master_ytcfg, webpage)
         identity_token = self._extract_identity_token(webpage, video_id)
 
-        player_responses = list(self._extract_player_responses(
+        player_responses, player_url = self._extract_player_responses(
             self._get_requested_clients(url, smuggled_data),
-            video_id, webpage, master_ytcfg, player_url, identity_token))
+            video_id, webpage, master_ytcfg, identity_token)
 
         get_first = lambda obj, keys, **kwargs: traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)