[instagram:user] Add pagination (closes #15934)

youtube_dl/extractor/instagram.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import itertools
 import json
 import re
 
@@ -242,18 +243,27 @@ class InstagramUserIE(InfoExtractor):
             return int_or_none(try_get(
                 node, lambda x: x['edge_media_' + suffix]['count']))
 
-        edges = self._download_json(
+        cursor = ''
-            'https://www.instagram.com/graphql/query/', uploader_id, query={
+        for page_num in itertools.count(1):
+            media = self._download_json(
+                'https://www.instagram.com/graphql/query/', uploader_id,
+                'Downloading JSON page %d' % page_num, query={
                     'query_hash': '472f257a40c653c64c666ce877d59d2b',
                     'variables': json.dumps({
                         'id': uploader_id,
-                    'first': 999999999,
+                        'first': 100,
+                        'after': cursor,
                     })
-            })['data']['user']['edge_owner_to_timeline_media']['edges']
+                })['data']['user']['edge_owner_to_timeline_media']
 
-        for edge in edges:
+            edges = media.get('edges')
-            node = edge['node']
+            if not edges or not isinstance(edges, list):
+                break
 
+            for edge in edges:
+                node = edge.get('node')
+                if not node or not isinstance(node, dict):
+                    continue
                 if node.get('__typename') != 'GraphVideo' and node.get('is_video') is not True:
                     continue
                 video_id = node.get('shortcode')
@@ -285,6 +295,18 @@ class InstagramUserIE(InfoExtractor):
 
                 yield info
 
+            page_info = media.get('page_info')
+            if not page_info or not isinstance(page_info, dict):
+                break
+
+            has_next_page = page_info.get('has_next_page')
+            if not has_next_page:
+                break
+
+            cursor = page_info.get('end_cursor')
+            if not cursor or not isinstance(cursor, compat_str):
+                break
+
     def _real_extract(self, url):
         username = self._match_id(url)
         uploader_id = self._download_json(