[loc] Improve (Closes #9521)

youtube_dl/extractor/libraryofcongress.py

@@ -3,63 +3,82 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    float_or_none,
+    int_or_none,
+)
 
 
 class LibraryOfCongressIE(InfoExtractor):
+    IE_NAME = 'loc'
+    IE_DESC = 'Library of Congress'
     _VALID_URL = r'https?://(?:www\.)?loc\.gov/item/(?P<id>[0-9]+)'
-    _TESTS = [{
+    _TEST = {
-    'url': 'http://loc.gov/item/90716351/',
+        'url': 'http://loc.gov/item/90716351/',
+        'md5': '353917ff7f0255aa6d4b80a034833de8',
         'info_dict': {
             'id': '90716351',
             'ext': 'mp4',
-            'title': 'Pa\'s trip to Mars /'
+            'title': "Pa's trip to Mars",
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 0,
+            'view_count': int,
         },
-        'params': {
+    }
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'https://www.loc.gov/item/97516576/',
-        'only_matching': True,
-    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
+        media_id = self._search_regex(
-        json_id = self._search_regex('media-player-([0-9A-Z]{32})', webpage, 'json id')
+            (r'id=(["\'])media-player-(?P<id>.+?)\1',
+             r'<video[^>]+id=(["\'])uuid-(?P<id>.+?)\1',
+             r'<video[^>]+data-uuid=(["\'])(?P<id>.+?)\1'),
+            webpage, 'media id', group='id')
+
+        data = self._parse_json(
+            self._download_webpage(
+                'https://media.loc.gov/services/v1/media?id=%s&context=json' % media_id,
+                video_id),
+            video_id)['mediaObject']
 
-        data = self._parse_json(self._download_webpage(
+        derivative = data['derivatives'][0]
-            'https://media.loc.gov/services/v1/media?id=%s' % json_id,
+        media_url = derivative['derivativeUrl']
-            video_id), video_id)
-        data = data['mediaObject']
 
-        media_url = data['derivatives'][0]['derivativeUrl']
+        # Following algorithm was extracted from setAVSource js function
+        # found in webpage
         media_url = media_url.replace('rtmp', 'https')
 
-        is_video = data['mediaType'].lower() == 'v'
+        is_video = data.get('mediaType', 'v').lower() == 'v'
-        if not determine_ext(media_url) in ('mp4', 'mp3'):
+        ext = determine_ext(media_url)
+        if ext not in ('mp4', 'mp3'):
             media_url += '.mp4' if is_video else '.mp3'
 
-        if media_url.index('vod/mp4:') > -1:
+        if 'vod/mp4:' in media_url:
-            media_url = media_url.replace('vod/mp4:', 'hls-vod/media/') + '.m3u8'
+            formats = [{
-        elif url.index('vod/mp3:') > -1:
+                'url': media_url.replace('vod/mp4:', 'hls-vod/media/') + '.m3u8',
-            media_url = media_url.replace('vod/mp3:', '')
+                'format_id': 'hls',
+                'ext': 'mp4',
+                'protocol': 'm3u8_native',
+            }]
+        elif 'vod/mp3:' in media_url:
+            formats = [{
+                'url': media_url.replace('vod/mp3:', ''),
+                'vcodec': 'none',
+            }]
+
+        self._sort_formats(formats)
 
-        formats = []
+        title = derivative.get('shortName') or data.get('shortName') or self._og_search_title(webpage)
-        if determine_ext(media_url) == 'm3u8':
+        duration = float_or_none(data.get('duration'))
-            formats = self._extract_m3u8_formats(media_url, video_id, ext='mp4')
+        view_count = int_or_none(data.get('viewCount'))
-        elif determine_ext(media_url) is 'mp3':
-            formats.append({
-                'url': media_url,
-                'ext': 'mp3',
-            })
 
         return {
             'id': video_id,
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'title': self._og_search_title(webpage),
+            'duration': duration,
+            'view_count': view_count,
             'formats': formats,
         }