From c645c7658d2a0cc13204b296dce3ec034ded5506 Mon Sep 17 00:00:00 2001 From: dst Date: Wed, 25 Dec 2013 08:34:41 +0700 Subject: [PATCH 1/2] [imdb] Extractor for lists (#2033) --- youtube_dl/extractor/__init__.py | 5 ++++- youtube_dl/extractor/imdb.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index a39a1e2f4..c96212760 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -79,7 +79,10 @@ from .hotnewhiphop import HotNewHipHopIE from .howcast import HowcastIE from .hypem import HypemIE from .ign import IGNIE, OneUPIE -from .imdb import ImdbIE +from .imdb import ( + ImdbIE, + ImdbListIE +) from .ina import InaIE from .infoq import InfoQIE from .instagram import InstagramIE diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py index e5332cce8..16926b4d3 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dl/extractor/imdb.py @@ -55,3 +55,32 @@ class ImdbIE(InfoExtractor): 'description': descr, 'thumbnail': format_info['slate'], } + +class ImdbListIE(InfoExtractor): + IE_NAME = u'imdb:list' + IE_DESC = u'Internet Movie Database lists' + _VALID_URL = r'http://www\.imdb\.com/list/(?P[\da-zA-Z_-]{11})' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + list_id = mobj.group('id') + + # RSS XML is sometimes malformed + rss = self._download_webpage('http://rss.imdb.com/list/%s' % list_id, list_id, u'Downloading list RSS') + list_title = self._html_search_regex(r'(.*?)', rss, u'list title') + + # Export is independent of actual author_id, but returns 404 if no author_id is provided. + # However, passing dummy author_id seems to be enough. + csv = self._download_webpage('http://www.imdb.com/list/export?list_id=%s&author_id=ur00000000' % list_id, + list_id, u'Downloading list CSV') + + entries = [] + for item in csv.split('\n')[1:]: + cols = item.split(',') + if len(cols) < 2: + continue + item_id = cols[1][1:-1] + if item_id.startswith('vi'): + entries.append(self.url_result('http://www.imdb.com/video/imdb/%s' % item_id, 'Imdb')) + + return self.playlist_result(entries, list_id, list_title) \ No newline at end of file From 41cc67c542046d300fc0a8a6bfb6c471db2db6b2 Mon Sep 17 00:00:00 2001 From: dst Date: Wed, 25 Dec 2013 08:40:09 +0700 Subject: [PATCH 2/2] [imdb] Add playlist test --- test/test_playlists.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/test_playlists.py b/test/test_playlists.py index 1b7b4e3d8..81352de6b 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -28,7 +28,8 @@ from youtube_dl.extractor import ( BandcampAlbumIE, SmotriCommunityIE, SmotriUserIE, - IviCompilationIE + IviCompilationIE, + ImdbListIE ) @@ -187,6 +188,15 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['id'], u'dezhurnyi_angel/season2') self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012) 2 сезон') self.assertTrue(len(result['entries']) >= 20) + + def test_imdb_list(self): + dl = FakeYDL() + ie = ImdbListIE(dl) + result = ie.extract('http://www.imdb.com/list/sMjedvGDd8U') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'sMjedvGDd8U') + self.assertEqual(result['title'], u'Animated and Family Films') + self.assertTrue(len(result['entries']) >= 48) if __name__ == '__main__':