From 627dcfff398b00429c8b310f1c1775f9be6f0268 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 28 Nov 2012 12:59:27 +0100 Subject: [PATCH 1/7] Restrict more characters (Closes #566) --- test/test_utils.py | 13 +++++++------ youtube_dl/utils.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index ceefabefc..fd8190f51 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -13,11 +13,12 @@ from youtube_dl.utils import sanitize_filename from youtube_dl.utils import unescapeHTML from youtube_dl.utils import orderedSet -if sys.version_info < (3,0): +if sys.version_info < (3, 0): _compat_str = lambda b: b.decode('unicode-escape') else: _compat_str = lambda s: s + class TestUtil(unittest.TestCase): def test_timeconvert(self): self.assertTrue(timeconvert('') is None) @@ -62,11 +63,11 @@ class TestUtil(unittest.TestCase): self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True)) self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True)) - tests =_compat_str('a\xe4b\u4e2d\u56fd\u7684c') + tests = _compat_str('a\xe4b\u4e2d\u56fd\u7684c') self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c') - self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '') # No empty filename + self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '') # No empty filename - forbidden = '"\0\\/&!: \'\t\n' + forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#' for fc in forbidden: for fbc in forbidden: self.assertTrue(fbc not in sanitize_filename(fc, restricted=True)) @@ -79,11 +80,11 @@ class TestUtil(unittest.TestCase): self.assertTrue(sanitize_filename(':', restricted=True) != '') def test_ordered_set(self): - self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7]) + self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) self.assertEqual(orderedSet([1]), [1]) #keep the list ordered - self.assertEqual(orderedSet([135,1,1,1]), [135,1]) + self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1]) def test_unescape_html(self): self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;')) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4600dc967..7f73b8476 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -330,7 +330,7 @@ def sanitize_filename(s, restricted=False): return '_-' if restricted else ' -' elif char in '\\/|*<>': return '_' - if restricted and (char in '!&\'' or char.isspace()): + if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()): return '_' if restricted and ord(char) > 127: return '_' From cdab8aa389a81b82ed3a2d72b05df5445d4b0668 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 28 Nov 2012 15:09:56 +0100 Subject: [PATCH 2/7] Update download tests --- test/gentests.py | 114 +++++++++++++++ test/test_download.py | 328 +++++++++++++++++++----------------------- test/tests.json | 63 ++++++++ 3 files changed, 327 insertions(+), 178 deletions(-) create mode 100755 test/gentests.py create mode 100644 test/tests.json diff --git a/test/gentests.py b/test/gentests.py new file mode 100755 index 000000000..8a76ff9f7 --- /dev/null +++ b/test/gentests.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +import io # for python 2 +import json +import os +import sys +import unittest + +# Allow direct execution +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import youtube_dl.InfoExtractors + +HEADER = u'''#!/usr/bin/env python + +# DO NOT EDIT THIS FILE BY HAND! +# It is auto-generated from tests.json and gentests.py. + +import hashlib +import io +import os +import json +import unittest +import sys + +# Allow direct execution +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl.FileDownloader import FileDownloader +import youtube_dl.InfoExtractors + +def _file_md5(fn): + with open(fn, 'rb') as f: + return hashlib.md5(f.read()).hexdigest() + +def md5_for_file(filename, block_size=2**20): + with open(filename) as f: + md5 = hashlib.md5() + while True: + data = f.read(block_size) + if not data: + break + md5.update(data) + return md5.hexdigest() +_file_md5 = md5_for_file + +class DownloadTest(unittest.TestCase): + PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") + + def setUp(self): + # Clear old files + self.tearDown() + + with io.open(self.PARAMETERS_FILE, encoding='utf-8') as pf: + self.parameters = json.load(pf) +''' + +FOOTER = u''' + +if __name__ == '__main__': + unittest.main() +''' + +DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json') +TEST_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test_download.py') + +def gentests(): + with io.open(DEF_FILE, encoding='utf-8') as deff: + defs = json.load(deff) + with io.open(TEST_FILE, 'w', encoding='utf-8') as testf: + testf.write(HEADER) + spaces = ' ' * 4 + write = lambda l: testf.write(spaces + l + '\n') + + for d in defs: + name = d['name'] + ie = getattr(youtube_dl.InfoExtractors, name + 'IE') + testf.write('\n') + if not ie._WORKING: + write('@unittest.skip("IE marked as not _WORKING")') + elif not d['file']: + write('@unittest.skip("No output file specified")') + elif 'skip' in d: + write('@unittest.skip(' + repr(d['skip']) + ')') + write('def test_' + name + '(self):') + write(' ' + name + 'IE = youtube_dl.InfoExtractors.' + name + 'IE') + write(' filename = ' + repr(d['file'])) + write(' fd = FileDownloader(self.parameters)') + write(' fd.add_info_extractor(' + name + 'IE())') + for ien in d.get('addIEs', []): + write(' fd.add_info_extractor(youtube_dl.InfoExtractors.' + ien + 'IE())') + write(' fd.download([' + repr(d['url']) + '])') + write(' self.assertTrue(os.path.exists(filename))') + if 'size' in d: + write(' self.assertEqual(os.path.getsize(filename), ' + repr(d['size']) + ')') + if 'md5' in d: + write(' md5_for_file = _file_md5(filename)') + write(' self.assertEqual(md5_for_file, ' + repr(d['md5']) + ')') + + testf.write('\n\n') + write('def tearDown(self):') + for d in defs: + if d['file']: + write(' if os.path.exists(' + repr(d['file']) + '):') + write(' os.remove(' + repr(d['file']) + ')') + else: + write(' # No file specified for ' + d['name']) + testf.write('\n') + testf.write(FOOTER) + +if __name__ == '__main__': + gentests() diff --git a/test/test_download.py b/test/test_download.py index b3fd7d992..92b552267 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -1,198 +1,170 @@ -#!/usr/bin/env python2 -import unittest +#!/usr/bin/env python + +# DO NOT EDIT THIS FILE BY HAND! +# It is auto-generated from tests.json and gentests.py. + import hashlib +import io import os import json +import unittest +import sys + +# Allow direct execution +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from youtube_dl.FileDownloader import FileDownloader -from youtube_dl.InfoExtractors import YoutubeIE, DailymotionIE -from youtube_dl.InfoExtractors import MetacafeIE, BlipTVIE -from youtube_dl.InfoExtractors import XVideosIE, VimeoIE -from youtube_dl.InfoExtractors import SoundcloudIE, StanfordOpenClassroomIE -from youtube_dl.InfoExtractors import CollegeHumorIE, XNXXIE +import youtube_dl.InfoExtractors +def _file_md5(fn): + with open(fn, 'rb') as f: + return hashlib.md5(f.read()).hexdigest() + +def md5_for_file(filename, block_size=2**20): + with open(filename) as f: + md5 = hashlib.md5() + while True: + data = f.read(block_size) + if not data: + break + md5.update(data) + return md5.hexdigest() +_file_md5 = md5_for_file class DownloadTest(unittest.TestCase): - PARAMETERS_FILE = "test/parameters.json" - #calculated with md5sum: - #md5sum (GNU coreutils) 8.19 - - YOUTUBE_SIZE = 1993883 - YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc" - YOUTUBE_FILE = "BaW_jenozKc.mp4" - - DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47" - DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech" - DAILYMOTION_FILE = "x33vw9.mp4" - - METACAFE_SIZE = 5754305 - METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/" - METACAFE_FILE = "_aUehQsCQtM.flv" - - BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7" - BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352" - BLIP_FILE = "5779306.m4v" - - XVIDEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf" - XVIDEO_URL = "http://www.xvideos.com/video939581/funny_porns_by_s_-1" - XVIDEO_FILE = "939581.flv" - - VIMEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf" - VIMEO_URL = "http://vimeo.com/14160053" - VIMEO_FILE = "" - - VIMEO2_MD5 = "" - VIMEO2_URL = "http://player.vimeo.com/video/47019590" - VIMEO2_FILE = "" - - SOUNDCLOUD_MD5 = "ce3775768ebb6432fa8495d446a078ed" - SOUNDCLOUD_URL = "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy" - SOUNDCLOUD_FILE = "n6FLbx6ZzMiu.mp3" - - STANDFORD_MD5 = "22c8206291368c4e2c9c1a307f0ea0f4" - STANDFORD_URL = "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100" - STANDFORD_FILE = "PracticalUnix_intro-environment.mp4" - - COLLEGEHUMOR_MD5 = "" - COLLEGEHUMOR_URL = "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody" - COLLEGEHUMOR_FILE = "" - - XNXX_MD5 = "5f0469c8d1dfd1bc38c8e6deb5e0a21d" - XNXX_URL = "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_" - XNXX_FILE = "1135332.flv" - - def test_youtube(self): - #let's download a file from youtube - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) - fd.add_info_extractor(YoutubeIE()) - fd.download([DownloadTest.YOUTUBE_URL]) - self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE)) - self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE) + PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") - def test_dailymotion(self): - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + def setUp(self): + # Clear old files + self.tearDown() + + with io.open(self.PARAMETERS_FILE, encoding='utf-8') as pf: + self.parameters = json.load(pf) + + def test_Youtube(self): + YoutubeIE = youtube_dl.InfoExtractors.YoutubeIE + filename = 'BaW_jenozKc.mp4' + fd = FileDownloader(self.parameters) + fd.add_info_extractor(YoutubeIE()) + fd.download(['http://www.youtube.com/watch?v=BaW_jenozKc']) + self.assertTrue(os.path.exists(filename)) + self.assertEqual(os.path.getsize(filename), 1993883) + + def test_Dailymotion(self): + DailymotionIE = youtube_dl.InfoExtractors.DailymotionIE + filename = 'x33vw9.mp4' + fd = FileDownloader(self.parameters) fd.add_info_extractor(DailymotionIE()) - fd.download([DownloadTest.DAILYMOTION_URL]) - self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE)) - md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE) - self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5) - - def test_metacafe(self): - #this emulate a skip,to be 2.6 compatible - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.download(['http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, 'd363a50e9eb4f22ce90d08d15695bb47') + + def test_Metacafe(self): + MetacafeIE = youtube_dl.InfoExtractors.MetacafeIE + filename = 'aUehQsCQtM.flv' + fd = FileDownloader(self.parameters) fd.add_info_extractor(MetacafeIE()) - fd.add_info_extractor(YoutubeIE()) - fd.download([DownloadTest.METACAFE_URL]) - self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE)) - self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE) - - def test_blip(self): - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE()) + fd.download(['http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/']) + self.assertTrue(os.path.exists(filename)) + self.assertEqual(os.path.getsize(filename), 5754305) + + def test_BlipTV(self): + BlipTVIE = youtube_dl.InfoExtractors.BlipTVIE + filename = '5779306.m4v' + fd = FileDownloader(self.parameters) fd.add_info_extractor(BlipTVIE()) - fd.download([DownloadTest.BLIP_URL]) - self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE)) - md5_down_file = md5_for_file(DownloadTest.BLIP_FILE) - self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5) - - def test_xvideo(self): - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.download(['http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, '93c24d2f4e0782af13b8a7606ea97ba7') + + def test_XVideos(self): + XVideosIE = youtube_dl.InfoExtractors.XVideosIE + filename = '939581.flv' + fd = FileDownloader(self.parameters) fd.add_info_extractor(XVideosIE()) - fd.download([DownloadTest.XVIDEO_URL]) - self.assertTrue(os.path.exists(DownloadTest.XVIDEO_FILE)) - md5_down_file = md5_for_file(DownloadTest.XVIDEO_FILE) - self.assertEqual(md5_down_file, DownloadTest.XVIDEO_MD5) - - def test_vimeo(self): - #skipped for the moment produce an error - return - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.download(['http://www.xvideos.com/video939581/funny_porns_by_s_-1']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, '1ab4dedc01f771cb2a65e91caa801aaf') + + @unittest.skip("No output file specified") + def test_Vimeo(self): + VimeoIE = youtube_dl.InfoExtractors.VimeoIE + filename = '' + fd = FileDownloader(self.parameters) fd.add_info_extractor(VimeoIE()) - fd.download([DownloadTest.VIMEO_URL]) - self.assertTrue(os.path.exists(DownloadTest.VIMEO_FILE)) - md5_down_file = md5_for_file(DownloadTest.VIMEO_FILE) - self.assertEqual(md5_down_file, DownloadTest.VIMEO_MD5) - - def test_vimeo2(self): - #skipped for the moment produce an error - return - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) - fd.add_info_extractor(VimeoIE()) - fd.download([DownloadTest.VIMEO2_URL]) - self.assertTrue(os.path.exists(DownloadTest.VIMEO2_FILE)) - md5_down_file = md5_for_file(DownloadTest.VIMEO2_FILE) - self.assertEqual(md5_down_file, DownloadTest.VIMEO2_MD5) - - def test_soundcloud(self): - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.download(['http://vimeo.com/14160053']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, '1ab4dedc01f771cb2a65e91caa801aaf') + + def test_Soundcloud(self): + SoundcloudIE = youtube_dl.InfoExtractors.SoundcloudIE + filename = 'n6FLbx6ZzMiu.mp3' + fd = FileDownloader(self.parameters) fd.add_info_extractor(SoundcloudIE()) - fd.download([DownloadTest.SOUNDCLOUD_URL]) - self.assertTrue(os.path.exists(DownloadTest.SOUNDCLOUD_FILE)) - md5_down_file = md5_for_file(DownloadTest.SOUNDCLOUD_FILE) - self.assertEqual(md5_down_file, DownloadTest.SOUNDCLOUD_MD5) - - def test_standford(self): - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, 'ce3775768ebb6432fa8495d446a078ed') + + def test_StanfordOpenClassroom(self): + StanfordOpenClassroomIE = youtube_dl.InfoExtractors.StanfordOpenClassroomIE + filename = 'PracticalUnix_intro-environment.mp4' + fd = FileDownloader(self.parameters) fd.add_info_extractor(StanfordOpenClassroomIE()) - fd.download([DownloadTest.STANDFORD_URL]) - self.assertTrue(os.path.exists(DownloadTest.STANDFORD_FILE)) - md5_down_file = md5_for_file(DownloadTest.STANDFORD_FILE) - self.assertEqual(md5_down_file, DownloadTest.STANDFORD_MD5) - - def test_collegehumor(self): - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.download(['http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, '22c8206291368c4e2c9c1a307f0ea0f4') + + @unittest.skip("IE marked as not _WORKING") + def test_CollegeHumor(self): + CollegeHumorIE = youtube_dl.InfoExtractors.CollegeHumorIE + filename = '' + fd = FileDownloader(self.parameters) fd.add_info_extractor(CollegeHumorIE()) - fd.download([DownloadTest.COLLEGEHUMOR_URL]) - self.assertTrue(os.path.exists(DownloadTest.COLLEGEHUMOR_FILE)) - md5_down_file = md5_for_file(DownloadTest.COLLEGEHUMOR_FILE) - self.assertEqual(md5_down_file, DownloadTest.COLLEGEHUMOR_MD5) - - def test_xnxx(self): - with open(DownloadTest.PARAMETERS_FILE) as f: - fd = FileDownloader(json.load(f)) + fd.download(['http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, '') + + def test_XNXX(self): + XNXXIE = youtube_dl.InfoExtractors.XNXXIE + filename = '1135332.flv' + fd = FileDownloader(self.parameters) fd.add_info_extractor(XNXXIE()) - fd.download([DownloadTest.XNXX_URL]) - self.assertTrue(os.path.exists(DownloadTest.XNXX_FILE)) - md5_down_file = md5_for_file(DownloadTest.XNXX_FILE) - self.assertEqual(md5_down_file, DownloadTest.XNXX_MD5) + fd.download(['http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, '5f0469c8d1dfd1bc38c8e6deb5e0a21d') - def tearDown(self): - if os.path.exists(DownloadTest.YOUTUBE_FILE): - os.remove(DownloadTest.YOUTUBE_FILE) - if os.path.exists(DownloadTest.DAILYMOTION_FILE): - os.remove(DownloadTest.DAILYMOTION_FILE) - if os.path.exists(DownloadTest.METACAFE_FILE): - os.remove(DownloadTest.METACAFE_FILE) - if os.path.exists(DownloadTest.BLIP_FILE): - os.remove(DownloadTest.BLIP_FILE) - if os.path.exists(DownloadTest.XVIDEO_FILE): - os.remove(DownloadTest.XVIDEO_FILE) - if os.path.exists(DownloadTest.VIMEO_FILE): - os.remove(DownloadTest.VIMEO_FILE) - if os.path.exists(DownloadTest.SOUNDCLOUD_FILE): - os.remove(DownloadTest.SOUNDCLOUD_FILE) - if os.path.exists(DownloadTest.STANDFORD_FILE): - os.remove(DownloadTest.STANDFORD_FILE) - if os.path.exists(DownloadTest.COLLEGEHUMOR_FILE): - os.remove(DownloadTest.COLLEGEHUMOR_FILE) - if os.path.exists(DownloadTest.XNXX_FILE): - os.remove(DownloadTest.XNXX_FILE) -def md5_for_file(filename, block_size=2**20): - with open(filename) as f: - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() + def tearDown(self): + if os.path.exists('BaW_jenozKc.mp4'): + os.remove('BaW_jenozKc.mp4') + if os.path.exists('x33vw9.mp4'): + os.remove('x33vw9.mp4') + if os.path.exists('aUehQsCQtM.flv'): + os.remove('aUehQsCQtM.flv') + if os.path.exists('5779306.m4v'): + os.remove('5779306.m4v') + if os.path.exists('939581.flv'): + os.remove('939581.flv') + # No file specified for Vimeo + if os.path.exists('n6FLbx6ZzMiu.mp3'): + os.remove('n6FLbx6ZzMiu.mp3') + if os.path.exists('PracticalUnix_intro-environment.mp4'): + os.remove('PracticalUnix_intro-environment.mp4') + # No file specified for CollegeHumor + if os.path.exists('1135332.flv'): + os.remove('1135332.flv') + + + +if __name__ == '__main__': + unittest.main() diff --git a/test/tests.json b/test/tests.json new file mode 100644 index 000000000..611780cc3 --- /dev/null +++ b/test/tests.json @@ -0,0 +1,63 @@ +[ + { + "name": "Youtube", + "size": 1993883, + "url": "http://www.youtube.com/watch?v=BaW_jenozKc", + "file": "BaW_jenozKc.mp4" + }, + { + "name": "Dailymotion", + "md5": "d363a50e9eb4f22ce90d08d15695bb47", + "url": "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech", + "file": "x33vw9.mp4" + }, + { + "name": "Metacafe", + "size": 5754305, + "addIEs": ["Youtube"], + "url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/", + "file": "aUehQsCQtM.flv" + }, + { + "name": "BlipTV", + "md5": "93c24d2f4e0782af13b8a7606ea97ba7", + "url": "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352", + "file": "5779306.m4v" + }, + { + "name": "XVideos", + "md5": "1ab4dedc01f771cb2a65e91caa801aaf", + "url": "http://www.xvideos.com/video939581/funny_porns_by_s_-1", + "file": "939581.flv" + }, + { + "name": "Vimeo", + "md5": "1ab4dedc01f771cb2a65e91caa801aaf", + "url": "http://vimeo.com/14160053", + "file": "" + }, + { + "name": "Soundcloud", + "md5": "ce3775768ebb6432fa8495d446a078ed", + "url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy", + "file": "n6FLbx6ZzMiu.mp3" + }, + { + "name": "StanfordOpenClassroom", + "md5": "22c8206291368c4e2c9c1a307f0ea0f4", + "url": "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100", + "file": "PracticalUnix_intro-environment.mp4" + }, + { + "name": "CollegeHumor", + "md5": "", + "url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody", + "file": "" + }, + { + "name": "XNXX", + "md5": "5f0469c8d1dfd1bc38c8e6deb5e0a21d", + "url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_", + "file": "1135332.flv" + } +] \ No newline at end of file From b08e09c3701d77413e1408a3475b513cc345cd5c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 28 Nov 2012 17:58:55 +0100 Subject: [PATCH 3/7] Mark broken IEs in --list-extractors --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index bf0ce14ec..37d6f8cfd 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -423,7 +423,7 @@ def _real_main(): if opts.list_extractors: for ie in extractors: - print(ie.IE_NAME) + print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) matchedUrls = filter(lambda url: ie.suitable(url), all_urls) all_urls = filter(lambda url: url not in matchedUrls, all_urls) for mu in matchedUrls: From 6ad98fb3fda767fb8da0d3c40da408ec2a09b5d4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 28 Nov 2012 18:21:06 +0100 Subject: [PATCH 4/7] Correct exception raising --- youtube_dl/FileDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index a135be352..48c8eb126 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -463,7 +463,7 @@ class FileDownloader(object): try: success = self._do_download(filename, info_dict) except (OSError, IOError) as err: - raise UnavailableVideoError + raise UnavailableVideoError() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.trouble(u'ERROR: unable to download video data: %s' % str(err)) return From 81760416054be7dcccc66b14d171872c8e13f183 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 28 Nov 2012 19:03:11 +0100 Subject: [PATCH 5/7] Check during test runtime instead of test generation for _WORKING, and add 2.6 compat --- test/gentests.py | 28 +++++++++++++++----- test/test_download.py | 60 +++++++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/test/gentests.py b/test/gentests.py index 8a76ff9f7..da4e3c2f6 100755 --- a/test/gentests.py +++ b/test/gentests.py @@ -46,6 +46,22 @@ def md5_for_file(filename, block_size=2**20): return md5.hexdigest() _file_md5 = md5_for_file + +try: + _skip_unless = unittest.skipUnless +except AttributeError: # Python 2.6 + def _skip_unless(cond, reason='No reason given'): + def resfunc(f): + def wfunc(*args, **kwargs): + if cond: + return f(*args, **kwargs) + else: + print('Skipped test') + return + return wfunc + return resfunc +_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs) + class DownloadTest(unittest.TestCase): PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") @@ -78,17 +94,15 @@ def gentests(): name = d['name'] ie = getattr(youtube_dl.InfoExtractors, name + 'IE') testf.write('\n') - if not ie._WORKING: - write('@unittest.skip("IE marked as not _WORKING")') - elif not d['file']: - write('@unittest.skip("No output file specified")') + write('@_skip_unless(youtube_dl.InfoExtractors.' + name + 'IE._WORKING, "IE marked as not _WORKING")') + if not d['file']: + write('@_skip("No output file specified")') elif 'skip' in d: - write('@unittest.skip(' + repr(d['skip']) + ')') + write('@_skip(' + repr(d['skip']) + ')') write('def test_' + name + '(self):') - write(' ' + name + 'IE = youtube_dl.InfoExtractors.' + name + 'IE') write(' filename = ' + repr(d['file'])) write(' fd = FileDownloader(self.parameters)') - write(' fd.add_info_extractor(' + name + 'IE())') + write(' fd.add_info_extractor(youtube_dl.InfoExtractors.' + name + 'IE())') for ien in d.get('addIEs', []): write(' fd.add_info_extractor(youtube_dl.InfoExtractors.' + ien + 'IE())') write(' fd.download([' + repr(d['url']) + '])') diff --git a/test/test_download.py b/test/test_download.py index 92b552267..0072d4aab 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -32,6 +32,22 @@ def md5_for_file(filename, block_size=2**20): return md5.hexdigest() _file_md5 = md5_for_file + +try: + _skip_unless = unittest.skipUnless +except AttributeError: # Python 2.6 + def _skip_unless(cond, reason='No reason given'): + def resfunc(f): + def wfunc(*args, **kwargs): + if cond: + return f(*args, **kwargs) + else: + print('Skipped test') + return + return wfunc + return resfunc +_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs) + class DownloadTest(unittest.TestCase): PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") @@ -42,102 +58,102 @@ class DownloadTest(unittest.TestCase): with io.open(self.PARAMETERS_FILE, encoding='utf-8') as pf: self.parameters = json.load(pf) + @_skip_unless(youtube_dl.InfoExtractors.YoutubeIE._WORKING, "IE marked as not _WORKING") def test_Youtube(self): - YoutubeIE = youtube_dl.InfoExtractors.YoutubeIE filename = 'BaW_jenozKc.mp4' fd = FileDownloader(self.parameters) - fd.add_info_extractor(YoutubeIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE()) fd.download(['http://www.youtube.com/watch?v=BaW_jenozKc']) self.assertTrue(os.path.exists(filename)) self.assertEqual(os.path.getsize(filename), 1993883) + @_skip_unless(youtube_dl.InfoExtractors.DailymotionIE._WORKING, "IE marked as not _WORKING") def test_Dailymotion(self): - DailymotionIE = youtube_dl.InfoExtractors.DailymotionIE filename = 'x33vw9.mp4' fd = FileDownloader(self.parameters) - fd.add_info_extractor(DailymotionIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.DailymotionIE()) fd.download(['http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, 'd363a50e9eb4f22ce90d08d15695bb47') + @_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING") def test_Metacafe(self): - MetacafeIE = youtube_dl.InfoExtractors.MetacafeIE filename = 'aUehQsCQtM.flv' fd = FileDownloader(self.parameters) - fd.add_info_extractor(MetacafeIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE()) fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE()) fd.download(['http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/']) self.assertTrue(os.path.exists(filename)) self.assertEqual(os.path.getsize(filename), 5754305) + @_skip_unless(youtube_dl.InfoExtractors.BlipTVIE._WORKING, "IE marked as not _WORKING") def test_BlipTV(self): - BlipTVIE = youtube_dl.InfoExtractors.BlipTVIE filename = '5779306.m4v' fd = FileDownloader(self.parameters) - fd.add_info_extractor(BlipTVIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.BlipTVIE()) fd.download(['http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, '93c24d2f4e0782af13b8a7606ea97ba7') + @_skip_unless(youtube_dl.InfoExtractors.XVideosIE._WORKING, "IE marked as not _WORKING") def test_XVideos(self): - XVideosIE = youtube_dl.InfoExtractors.XVideosIE filename = '939581.flv' fd = FileDownloader(self.parameters) - fd.add_info_extractor(XVideosIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.XVideosIE()) fd.download(['http://www.xvideos.com/video939581/funny_porns_by_s_-1']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, '1ab4dedc01f771cb2a65e91caa801aaf') - @unittest.skip("No output file specified") + @_skip_unless(youtube_dl.InfoExtractors.VimeoIE._WORKING, "IE marked as not _WORKING") + @_skip("No output file specified") def test_Vimeo(self): - VimeoIE = youtube_dl.InfoExtractors.VimeoIE filename = '' fd = FileDownloader(self.parameters) - fd.add_info_extractor(VimeoIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.VimeoIE()) fd.download(['http://vimeo.com/14160053']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, '1ab4dedc01f771cb2a65e91caa801aaf') + @_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING") def test_Soundcloud(self): - SoundcloudIE = youtube_dl.InfoExtractors.SoundcloudIE filename = 'n6FLbx6ZzMiu.mp3' fd = FileDownloader(self.parameters) - fd.add_info_extractor(SoundcloudIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE()) fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, 'ce3775768ebb6432fa8495d446a078ed') + @_skip_unless(youtube_dl.InfoExtractors.StanfordOpenClassroomIE._WORKING, "IE marked as not _WORKING") def test_StanfordOpenClassroom(self): - StanfordOpenClassroomIE = youtube_dl.InfoExtractors.StanfordOpenClassroomIE filename = 'PracticalUnix_intro-environment.mp4' fd = FileDownloader(self.parameters) - fd.add_info_extractor(StanfordOpenClassroomIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.StanfordOpenClassroomIE()) fd.download(['http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, '22c8206291368c4e2c9c1a307f0ea0f4') - @unittest.skip("IE marked as not _WORKING") + @_skip_unless(youtube_dl.InfoExtractors.CollegeHumorIE._WORKING, "IE marked as not _WORKING") + @_skip("No output file specified") def test_CollegeHumor(self): - CollegeHumorIE = youtube_dl.InfoExtractors.CollegeHumorIE filename = '' fd = FileDownloader(self.parameters) - fd.add_info_extractor(CollegeHumorIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.CollegeHumorIE()) fd.download(['http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, '') + @_skip_unless(youtube_dl.InfoExtractors.XNXXIE._WORKING, "IE marked as not _WORKING") def test_XNXX(self): - XNXXIE = youtube_dl.InfoExtractors.XNXXIE filename = '1135332.flv' fd = FileDownloader(self.parameters) - fd.add_info_extractor(XNXXIE()) + fd.add_info_extractor(youtube_dl.InfoExtractors.XNXXIE()) fd.download(['http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) From 743b28ce1155df348bbb828fcb316d57c8704328 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 29 Nov 2012 15:11:24 +0100 Subject: [PATCH 6/7] Allow youtube_dl/__main__.py to be called directly --- youtube_dl/__main__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/youtube_dl/__main__.py b/youtube_dl/__main__.py index ccc782bec..0910e9d15 100755 --- a/youtube_dl/__main__.py +++ b/youtube_dl/__main__.py @@ -1,6 +1,14 @@ #!/usr/bin/env python -# Execute with python -m youtube_dl +# Execute with +# $ python youtube_dl/__main__.py (2.6+) +# $ python -m youtube_dl (2.7+) + +import sys + +if __package__ is None: + import os.path + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import youtube_dl From 20ba04267c3c2be27a766a43f34014012799b761 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Thu, 29 Nov 2012 15:20:20 +0100 Subject: [PATCH 7/7] removed __main__.py from the root of the repo --- Makefile | 3 ++- __main__.py | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) delete mode 100755 __main__.py diff --git a/Makefile b/Makefile index c4937a512..2eb226fdb 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,8 @@ test: # TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them youtube-dl: youtube_dl/*.py - zip --quiet youtube-dl __main__.py youtube_dl/*.py + zip --quiet youtube-dl youtube_dl/*.py + zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py echo '#!/usr/bin/env python' > youtube-dl cat youtube-dl.zip >> youtube-dl rm youtube-dl.zip diff --git a/__main__.py b/__main__.py deleted file mode 100755 index fc3cc8ad8..000000000 --- a/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -import youtube_dl - -if __name__ == '__main__': - youtube_dl.main()