Merge remote-tracking branch 'upstream/master'

pull/8/head
Pierre Rudloff 11 years ago
commit 5c6658d4dd

@ -120,18 +120,20 @@ which means you can modify it, redistribute it or use it however you like.
--max-quality FORMAT highest quality format to download --max-quality FORMAT highest quality format to download
-F, --list-formats list all available formats (currently youtube -F, --list-formats list all available formats (currently youtube
only) only)
## Subtitle Options:
--write-sub write subtitle file (currently youtube only) --write-sub write subtitle file (currently youtube only)
--write-auto-sub write automatic subtitle file (currently youtube --write-auto-sub write automatic subtitle file (currently youtube
only) only)
--only-sub [deprecated] alias of --skip-download --only-sub [deprecated] alias of --skip-download
--all-subs downloads all the available subtitles of the --all-subs downloads all the available subtitles of the
video (currently youtube only) video
--list-subs lists all available subtitles for the video --list-subs lists all available subtitles for the video
(currently youtube only) --sub-format FORMAT subtitle format (default=srt) ([sbv/vtt] youtube
--sub-format FORMAT subtitle format [srt/sbv/vtt] (default=srt) only)
(currently youtube only) --sub-lang LANGS languages of the subtitles to download (optional)
--sub-lang LANG language of the subtitles to download (optional) separated by commas, use IETF language tags like
use IETF language tags like 'en' 'en,pt'
## Authentication Options: ## Authentication Options:
-u, --username USERNAME account username -u, --username USERNAME account username
@ -153,6 +155,8 @@ which means you can modify it, redistribute it or use it however you like.
processing; the video is erased by default processing; the video is erased by default
--no-post-overwrites do not overwrite post-processed files; the post- --no-post-overwrites do not overwrite post-processed files; the post-
processed files are overwritten by default processed files are overwritten by default
--embed-subs embed subtitles in the video (only for mp4
videos)
# CONFIGURATION # CONFIGURATION

@ -6,28 +6,32 @@ import hashlib
import urllib.request import urllib.request
if len(sys.argv) <= 1: if len(sys.argv) <= 1:
print('Specify the version number as parameter') print('Specify the version number as parameter')
sys.exit() sys.exit()
version = sys.argv[1] version = sys.argv[1]
with open('update/LATEST_VERSION', 'w') as f: with open('update/LATEST_VERSION', 'w') as f:
f.write(version) f.write(version)
versions_info = json.load(open('update/versions.json')) versions_info = json.load(open('update/versions.json'))
if 'signature' in versions_info: if 'signature' in versions_info:
del versions_info['signature'] del versions_info['signature']
new_version = {} new_version = {}
filenames = {'bin': 'youtube-dl', 'exe': 'youtube-dl.exe', 'tar': 'youtube-dl-%s.tar.gz' % version} filenames = {
'bin': 'youtube-dl',
'exe': 'youtube-dl.exe',
'tar': 'youtube-dl-%s.tar.gz' % version}
for key, filename in filenames.items(): for key, filename in filenames.items():
print('Downloading and checksumming %s...' %filename) print('Downloading and checksumming %s...' % filename)
url = 'http://youtube-dl.org/downloads/%s/%s' % (version, filename) url = 'https://yt-dl.org/downloads/%s/%s' % (version, filename)
data = urllib.request.urlopen(url).read() data = urllib.request.urlopen(url).read()
sha256sum = hashlib.sha256(data).hexdigest() sha256sum = hashlib.sha256(data).hexdigest()
new_version[key] = (url, sha256sum) new_version[key] = (url, sha256sum)
versions_info['versions'][version] = new_version versions_info['versions'][version] = new_version
versions_info['latest'] = version versions_info['latest'] = version
json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True) with open('update/versions.json', 'w') as jsonf:
json.dump(versions_info, jsonf, indent=4, sort_keys=True)

@ -22,7 +22,7 @@ entry_template=textwrap.dedent("""
<atom:link href="http://rg3.github.io/youtube-dl" /> <atom:link href="http://rg3.github.io/youtube-dl" />
<atom:content type="xhtml"> <atom:content type="xhtml">
<div xmlns="http://www.w3.org/1999/xhtml"> <div xmlns="http://www.w3.org/1999/xhtml">
Downloads available at <a href="http://youtube-dl.org/downloads/@VERSION@/">http://youtube-dl.org/downloads/@VERSION@/</a> Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a>
</div> </div>
</atom:content> </atom:content>
<atom:author> <atom:author>
@ -54,4 +54,3 @@ atom_template = atom_template.replace('@ENTRIES@', entries_str)
with open('update/releases.atom','w',encoding='utf-8') as atom_file: with open('update/releases.atom','w',encoding='utf-8') as atom_file:
atom_file.write(atom_template) atom_file.write(atom_template)

@ -67,7 +67,7 @@ RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz"
(cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS) (cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
git checkout HEAD -- youtube-dl youtube-dl.exe git checkout HEAD -- youtube-dl youtube-dl.exe
/bin/echo -e "\n### Signing and uploading the new binaries to youtube-dl.org..." /bin/echo -e "\n### Signing and uploading the new binaries to yt-dl.org ..."
for f in $RELEASE_FILES; do gpg --detach-sig "build/$version/$f"; done for f in $RELEASE_FILES; do gpg --detach-sig "build/$version/$f"; done
scp -r "build/$version" ytdl@yt-dl.org:html/tmp/ scp -r "build/$version" ytdl@yt-dl.org:html/tmp/
ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/" ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/"

@ -11,30 +11,36 @@ tests = [
# 90 # 90
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`",
"mrtyuioplkjhgfdsazxcvbne1234567890QWER[YUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={`]}|"), "mrtyuioplkjhgfdsazxcvbne1234567890QWER[YUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={`]}|"),
# 89
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'",
"/?;:|}<[{=+-_)(*&^%$#@!MqBVCXZASDFGHJKLPOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuyt"),
# 88 # 88
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<",
"J:|}][{=+-_)(*&;%$#@>MNBVCXZASDFGH^KLPOIUYTREWQ0987654321mnbvcxzasdfghrklpoiuytej"), "J:|}][{=+-_)(*&;%$#@>MNBVCXZASDFGH^KLPOIUYTREWQ0987654321mnbvcxzasdfghrklpoiuytej"),
# 87 - vflART1Nf 2013/07/24 # 87
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$^&*()_-+={[]}|:;?/>.<", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$^&*()_-+={[]}|:;?/>.<",
"tyuioplkjhgfdsazxcv<nm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$^&*()_-+={[]}|:;?/>"), "uioplkjhgfdsazxcvbnm1t34567890QWE2TYUIOPLKJHGFDSAZXCVeNM!@#$^&*()_-+={[]}|:;?/>.<"),
# 86 - vflm_D8eE 2013/07/31 # 86 - vflh9ybst 2013/08/23
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<",
">.1}|[{=+-_)(*&^%$#@!MNBVCXZASDFGHJK<POIUYTREW509876L432/mnbvcxzasdfghjklpoiuytre"), "yuioplkjhgfdsazxcvbnm1234567890QWERrYUIOPLKqHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<"),
# 85 - vflSAFCP9 2013/07/19 # 85
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?/>.<", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?/>.<",
"ertyuiqplkjhgfdsazx$vbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#<%^&*()_-+={[};?/c"), ".>/?;}[{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWQ0q876543r1mnbvcx9asdfghjklpoiuyt2"),
# 84 # 84 - vflh9ybst 2013/08/23 (sporadic)
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?>.<", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?>.<",
"<.>?;}[{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWe098765432rmnbvcxzasdfghjklpoiuyt1"), "yuioplkjhgfdsazxcvbnm1234567890QWERrYUIOPLKqHGFDSAZXCVBNM!@#$%^&*()_-+={[};?>.<"),
# 83 - vflTWC9KW 2013/08/01 # 83
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!#$%^&*()_+={[};?/>.<", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!#$%^&*()_+={[};?/>.<",
"qwertyuioplkjhg>dsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!#$%^&*()_+={[};?/f"), ".>/?;}[{=+_)(*&^%<#!MNBVCXZASPFGHJKLwOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuytreq"),
# 82 # 82 - vflZK4ZYR 2013/08/23
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.<", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.<",
"Q>/?;}[{=+-(*<^%$#@!MNBVCXZASDFGHKLPOIUY8REWT0q&7654321mnbvcxzasdfghjklpoiuytrew9"), "wertyuioplkjhgfdsaqxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&z(-+={[};?/>.<"),
# 81 - vflLC8JvQ 2013/07/25 # 81 - vflLC8JvQ 2013/07/25
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.",
"C>/?;}[{=+-(*&^%$#@!MNBVYXZASDFGHKLPOIU.TREWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"), "C>/?;}[{=+-(*&^%$#@!MNBVYXZASDFGHKLPOIU.TREWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),
# 80 - vflZK4ZYR 2013/08/23 (sporadic)
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>",
"wertyuioplkjhgfdsaqxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&z(-+={[};?/>"),
# 79 - vflLC8JvQ 2013/07/25 (sporadic) # 79 - vflLC8JvQ 2013/07/25 (sporadic)
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/", ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/",
"Z?;}[{=+-(*&^%$#@!MNBVCXRASDFGHKLPOIUYT/EWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"), "Z?;}[{=+-(*&^%$#@!MNBVCXRASDFGHKLPOIUYT/EWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),

@ -35,47 +35,47 @@ class TestYoutubeSubtitles(unittest.TestCase):
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260') self.assertEqual(md5(sub), '4cd9278a35ba2305f47354ee13472260')
def test_youtube_subtitles_it(self): def test_youtube_subtitles_it(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['subtitleslang'] = 'it' DL.params['subtitleslangs'] = ['it']
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['it']
self.assertEqual(md5(sub[2]), '164a51f16f260476a05b50fe4c2f161d') self.assertEqual(md5(sub), '164a51f16f260476a05b50fe4c2f161d')
def test_youtube_onlysubtitles(self): def test_youtube_onlysubtitles(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['onlysubtitles'] = True DL.params['onlysubtitles'] = True
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260') self.assertEqual(md5(sub), '4cd9278a35ba2305f47354ee13472260')
def test_youtube_allsubtitles(self): def test_youtube_allsubtitles(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['allsubtitles'] = True DL.params['allsubtitles'] = True
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
subtitles = info_dict[0]['subtitles'] subtitles = info_dict[0]['subtitles']
self.assertEqual(len(subtitles), 13) self.assertEqual(len(subtitles.keys()), 13)
def test_youtube_subtitles_sbv_format(self): def test_youtube_subtitles_sbv_format(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['subtitlesformat'] = 'sbv' DL.params['subtitlesformat'] = 'sbv'
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '13aeaa0c245a8bed9a451cb643e3ad8b') self.assertEqual(md5(sub), '13aeaa0c245a8bed9a451cb643e3ad8b')
def test_youtube_subtitles_vtt_format(self): def test_youtube_subtitles_vtt_format(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['subtitlesformat'] = 'vtt' DL.params['subtitlesformat'] = 'vtt'
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '356cdc577fde0c6783b9b822e7206ff7') self.assertEqual(md5(sub), '356cdc577fde0c6783b9b822e7206ff7')
def test_youtube_list_subtitles(self): def test_youtube_list_subtitles(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['listsubtitles'] = True DL.params['listsubtitles'] = True
@ -85,11 +85,20 @@ class TestYoutubeSubtitles(unittest.TestCase):
def test_youtube_automatic_captions(self): def test_youtube_automatic_captions(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writeautomaticsub'] = True DL.params['writeautomaticsub'] = True
DL.params['subtitleslang'] = 'it' DL.params['subtitleslangs'] = ['it']
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('8YoUxe5ncPo') info_dict = IE.extract('8YoUxe5ncPo')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['it']
self.assertTrue(sub[2] is not None) self.assertTrue(sub is not None)
def test_youtube_multiple_langs(self):
DL = FakeYDL()
DL.params['writesubtitles'] = True
langs = ['it', 'fr', 'de']
DL.params['subtitleslangs'] = langs
IE = YoutubeIE(DL)
subtitles = IE.extract('QRS8MkLhQmM')[0]['subtitles']
for lang in langs:
self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

@ -71,12 +71,17 @@ class FFmpegPostProcessor(PostProcessor):
programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe'] programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
return dict((program, executable(program)) for program in programs) return dict((program, executable(program)) for program in programs)
def run_ffmpeg(self, path, out_path, opts): def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
if not self._exes['ffmpeg'] and not self._exes['avconv']: if not self._exes['ffmpeg'] and not self._exes['avconv']:
raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.') raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path)]
files_cmd = []
for path in input_paths:
files_cmd.extend(['-i', encodeFilename(path)])
cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y'] + files_cmd
+ opts + + opts +
[encodeFilename(self._ffmpeg_filename_argument(out_path))]) [encodeFilename(self._ffmpeg_filename_argument(out_path))])
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout,stderr = p.communicate() stdout,stderr = p.communicate()
if p.returncode != 0: if p.returncode != 0:
@ -84,6 +89,9 @@ class FFmpegPostProcessor(PostProcessor):
msg = stderr.strip().split('\n')[-1] msg = stderr.strip().split('\n')[-1]
raise FFmpegPostProcessorError(msg) raise FFmpegPostProcessorError(msg)
def run_ffmpeg(self, path, out_path, opts):
self.run_ffmpeg_multiple_files([path], out_path, opts)
def _ffmpeg_filename_argument(self, fn): def _ffmpeg_filename_argument(self, fn):
# ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
if fn.startswith(u'-'): if fn.startswith(u'-'):
@ -232,3 +240,227 @@ class FFmpegVideoConvertor(FFmpegPostProcessor):
information['format'] = self._preferedformat information['format'] = self._preferedformat
information['ext'] = self._preferedformat information['ext'] = self._preferedformat
return False,information return False,information
class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
# See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
_lang_map = {
'aa': 'aar',
'ab': 'abk',
'ae': 'ave',
'af': 'afr',
'ak': 'aka',
'am': 'amh',
'an': 'arg',
'ar': 'ara',
'as': 'asm',
'av': 'ava',
'ay': 'aym',
'az': 'aze',
'ba': 'bak',
'be': 'bel',
'bg': 'bul',
'bh': 'bih',
'bi': 'bis',
'bm': 'bam',
'bn': 'ben',
'bo': 'bod',
'br': 'bre',
'bs': 'bos',
'ca': 'cat',
'ce': 'che',
'ch': 'cha',
'co': 'cos',
'cr': 'cre',
'cs': 'ces',
'cu': 'chu',
'cv': 'chv',
'cy': 'cym',
'da': 'dan',
'de': 'deu',
'dv': 'div',
'dz': 'dzo',
'ee': 'ewe',
'el': 'ell',
'en': 'eng',
'eo': 'epo',
'es': 'spa',
'et': 'est',
'eu': 'eus',
'fa': 'fas',
'ff': 'ful',
'fi': 'fin',
'fj': 'fij',
'fo': 'fao',
'fr': 'fra',
'fy': 'fry',
'ga': 'gle',
'gd': 'gla',
'gl': 'glg',
'gn': 'grn',
'gu': 'guj',
'gv': 'glv',
'ha': 'hau',
'he': 'heb',
'hi': 'hin',
'ho': 'hmo',
'hr': 'hrv',
'ht': 'hat',
'hu': 'hun',
'hy': 'hye',
'hz': 'her',
'ia': 'ina',
'id': 'ind',
'ie': 'ile',
'ig': 'ibo',
'ii': 'iii',
'ik': 'ipk',
'io': 'ido',
'is': 'isl',
'it': 'ita',
'iu': 'iku',
'ja': 'jpn',
'jv': 'jav',
'ka': 'kat',
'kg': 'kon',
'ki': 'kik',
'kj': 'kua',
'kk': 'kaz',
'kl': 'kal',
'km': 'khm',
'kn': 'kan',
'ko': 'kor',
'kr': 'kau',
'ks': 'kas',
'ku': 'kur',
'kv': 'kom',
'kw': 'cor',
'ky': 'kir',
'la': 'lat',
'lb': 'ltz',
'lg': 'lug',
'li': 'lim',
'ln': 'lin',
'lo': 'lao',
'lt': 'lit',
'lu': 'lub',
'lv': 'lav',
'mg': 'mlg',
'mh': 'mah',
'mi': 'mri',
'mk': 'mkd',
'ml': 'mal',
'mn': 'mon',
'mr': 'mar',
'ms': 'msa',
'mt': 'mlt',
'my': 'mya',
'na': 'nau',
'nb': 'nob',
'nd': 'nde',
'ne': 'nep',
'ng': 'ndo',
'nl': 'nld',
'nn': 'nno',
'no': 'nor',
'nr': 'nbl',
'nv': 'nav',
'ny': 'nya',
'oc': 'oci',
'oj': 'oji',
'om': 'orm',
'or': 'ori',
'os': 'oss',
'pa': 'pan',
'pi': 'pli',
'pl': 'pol',
'ps': 'pus',
'pt': 'por',
'qu': 'que',
'rm': 'roh',
'rn': 'run',
'ro': 'ron',
'ru': 'rus',
'rw': 'kin',
'sa': 'san',
'sc': 'srd',
'sd': 'snd',
'se': 'sme',
'sg': 'sag',
'si': 'sin',
'sk': 'slk',
'sl': 'slv',
'sm': 'smo',
'sn': 'sna',
'so': 'som',
'sq': 'sqi',
'sr': 'srp',
'ss': 'ssw',
'st': 'sot',
'su': 'sun',
'sv': 'swe',
'sw': 'swa',
'ta': 'tam',
'te': 'tel',
'tg': 'tgk',
'th': 'tha',
'ti': 'tir',
'tk': 'tuk',
'tl': 'tgl',
'tn': 'tsn',
'to': 'ton',
'tr': 'tur',
'ts': 'tso',
'tt': 'tat',
'tw': 'twi',
'ty': 'tah',
'ug': 'uig',
'uk': 'ukr',
'ur': 'urd',
'uz': 'uzb',
've': 'ven',
'vi': 'vie',
'vo': 'vol',
'wa': 'wln',
'wo': 'wol',
'xh': 'xho',
'yi': 'yid',
'yo': 'yor',
'za': 'zha',
'zh': 'zho',
'zu': 'zul',
}
def __init__(self, downloader=None, subtitlesformat='srt'):
super(FFmpegEmbedSubtitlePP, self).__init__(downloader)
self._subformat = subtitlesformat
@classmethod
def _conver_lang_code(cls, code):
"""Convert language code from ISO 639-1 to ISO 639-2/T"""
return cls._lang_map.get(code[:2])
def run(self, information):
if information['ext'] != u'mp4':
self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
return True, information
sub_langs = [key for key in information['subtitles']]
filename = information['filepath']
input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
opts = ['-map', '0:0', '-map', '0:1', '-c:v', 'copy', '-c:a', 'copy']
for (i, lang) in enumerate(sub_langs):
opts.extend(['-map', '%d:0' % (i+1), '-c:s:%d' % i, 'mov_text'])
lang_code = self._conver_lang_code(lang)
if lang_code is not None:
opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
opts.extend(['-f', 'mp4'])
temp_filename = filename + u'.temp'
self._downloader.to_screen(u'[ffmpeg] Embedding subtitles in \'%s\'' % filename)
self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
os.remove(encodeFilename(filename))
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
return True, information

@ -76,7 +76,7 @@ class YoutubeDL(object):
allsubtitles: Downloads all the subtitles of the video allsubtitles: Downloads all the subtitles of the video
listsubtitles: Lists all available subtitles for the video listsubtitles: Lists all available subtitles for the video
subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt) subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
subtitleslang: Language of the subtitles to download subtitleslangs: List of languages of the subtitles to download
keepvideo: Keep the video file after post-processing keepvideo: Keep the video file after post-processing
daterange: A DateRange object, download only if the upload_date is in the range. daterange: A DateRange object, download only if the upload_date is in the range.
skip_download: Skip the actual download of the video file skip_download: Skip the actual download of the video file
@ -483,41 +483,28 @@ class YoutubeDL(object):
self.report_error(u'Cannot write description file ' + descfn) self.report_error(u'Cannot write description file ' + descfn)
return return
if (self.params.get('writesubtitles', False) or self.params.get('writeautomaticsub')) and 'subtitles' in info_dict and info_dict['subtitles']: subtitles_are_requested = any([self.params.get('writesubtitles', False),
self.params.get('writeautomaticsub'),
self.params.get('allsubtitles', False)])
if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
# subtitles download errors are already managed as troubles in relevant IE # subtitles download errors are already managed as troubles in relevant IE
# that way it will silently go on when used with unsupporting IE # that way it will silently go on when used with unsupporting IE
subtitle = info_dict['subtitles'][0] subtitles = info_dict['subtitles']
(sub_error, sub_lang, sub) = subtitle
sub_format = self.params.get('subtitlesformat') sub_format = self.params.get('subtitlesformat')
if sub_error: for sub_lang in subtitles.keys():
self.report_warning("Some error while getting the subtitles") sub = subtitles[sub_lang]
else: if sub is None:
continue
try: try:
sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format sub_filename = subtitles_filename(filename, sub_lang, sub_format)
self.report_writesubtitles(sub_filename) self.report_writesubtitles(sub_filename)
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
subfile.write(sub) subfile.write(sub)
except (OSError, IOError): except (OSError, IOError):
self.report_error(u'Cannot write subtitles file ' + descfn) self.report_error(u'Cannot write subtitles file ' + descfn)
return return
if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
subtitles = info_dict['subtitles']
sub_format = self.params.get('subtitlesformat')
for subtitle in subtitles:
(sub_error, sub_lang, sub) = subtitle
if sub_error:
self.report_warning("Some error while getting the subtitles")
else:
try:
sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
self.report_writesubtitles(sub_filename)
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
subfile.write(sub)
except (OSError, IOError):
self.report_error(u'Cannot write subtitles file ' + descfn)
return
if self.params.get('writeinfojson', False): if self.params.get('writeinfojson', False):
infofn = filename + u'.info.json' infofn = filename + u'.info.json'
self.report_writeinfojson(infofn) self.report_writeinfojson(infofn)

@ -83,6 +83,9 @@ def parseOpts(overrideArguments=None):
return "".join(opts) return "".join(opts)
def _comma_separated_values_options_callback(option, opt_str, value, parser):
setattr(parser.values, option.dest, value.split(','))
def _find_term_columns(): def _find_term_columns():
columns = os.environ.get('COLUMNS', None) columns = os.environ.get('COLUMNS', None)
if columns: if columns:
@ -120,6 +123,7 @@ def parseOpts(overrideArguments=None):
selection = optparse.OptionGroup(parser, 'Video Selection') selection = optparse.OptionGroup(parser, 'Video Selection')
authentication = optparse.OptionGroup(parser, 'Authentication Options') authentication = optparse.OptionGroup(parser, 'Authentication Options')
video_format = optparse.OptionGroup(parser, 'Video Format Options') video_format = optparse.OptionGroup(parser, 'Video Format Options')
subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
downloader = optparse.OptionGroup(parser, 'Download Options') downloader = optparse.OptionGroup(parser, 'Download Options')
postproc = optparse.OptionGroup(parser, 'Post-processing Options') postproc = optparse.OptionGroup(parser, 'Post-processing Options')
filesystem = optparse.OptionGroup(parser, 'Filesystem Options') filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
@ -186,27 +190,29 @@ def parseOpts(overrideArguments=None):
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
video_format.add_option('-F', '--list-formats', video_format.add_option('-F', '--list-formats',
action='store_true', dest='listformats', help='list all available formats (currently youtube only)') action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
video_format.add_option('--write-sub', '--write-srt',
subtitles.add_option('--write-sub', '--write-srt',
action='store_true', dest='writesubtitles', action='store_true', dest='writesubtitles',
help='write subtitle file (currently youtube only)', default=False) help='write subtitle file (currently youtube only)', default=False)
video_format.add_option('--write-auto-sub', '--write-automatic-sub', subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
action='store_true', dest='writeautomaticsub', action='store_true', dest='writeautomaticsub',
help='write automatic subtitle file (currently youtube only)', default=False) help='write automatic subtitle file (currently youtube only)', default=False)
video_format.add_option('--only-sub', subtitles.add_option('--only-sub',
action='store_true', dest='skip_download', action='store_true', dest='skip_download',
help='[deprecated] alias of --skip-download', default=False) help='[deprecated] alias of --skip-download', default=False)
video_format.add_option('--all-subs', subtitles.add_option('--all-subs',
action='store_true', dest='allsubtitles', action='store_true', dest='allsubtitles',
help='downloads all the available subtitles of the video (currently youtube only)', default=False) help='downloads all the available subtitles of the video', default=False)
video_format.add_option('--list-subs', subtitles.add_option('--list-subs',
action='store_true', dest='listsubtitles', action='store_true', dest='listsubtitles',
help='lists all available subtitles for the video (currently youtube only)', default=False) help='lists all available subtitles for the video', default=False)
video_format.add_option('--sub-format', subtitles.add_option('--sub-format',
action='store', dest='subtitlesformat', metavar='FORMAT', action='store', dest='subtitlesformat', metavar='FORMAT',
help='subtitle format [srt/sbv/vtt] (default=srt) (currently youtube only)', default='srt') help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
video_format.add_option('--sub-lang', '--srt-lang', subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
action='store', dest='subtitleslang', metavar='LANG', action='callback', dest='subtitleslang', metavar='LANGS', type='str',
help='language of the subtitles to download (optional) use IETF language tags like \'en\'') default=[], callback=_comma_separated_values_options_callback,
help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
downloader.add_option('-r', '--rate-limit', downloader.add_option('-r', '--rate-limit',
dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)') dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)')
@ -321,6 +327,8 @@ def parseOpts(overrideArguments=None):
help='keeps the video file on disk after the post-processing; the video is erased by default') help='keeps the video file on disk after the post-processing; the video is erased by default')
postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
help='do not overwrite post-processed files; the post-processed files are overwritten by default') help='do not overwrite post-processed files; the post-processed files are overwritten by default')
postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
help='embed subtitles in the video (only for mp4 videos)')
parser.add_option_group(general) parser.add_option_group(general)
@ -329,6 +337,7 @@ def parseOpts(overrideArguments=None):
parser.add_option_group(filesystem) parser.add_option_group(filesystem)
parser.add_option_group(verbosity) parser.add_option_group(verbosity)
parser.add_option_group(video_format) parser.add_option_group(video_format)
parser.add_option_group(subtitles)
parser.add_option_group(authentication) parser.add_option_group(authentication)
parser.add_option_group(postproc) parser.add_option_group(postproc)
@ -568,7 +577,7 @@ def _real_main(argv=None):
'allsubtitles': opts.allsubtitles, 'allsubtitles': opts.allsubtitles,
'listsubtitles': opts.listsubtitles, 'listsubtitles': opts.listsubtitles,
'subtitlesformat': opts.subtitlesformat, 'subtitlesformat': opts.subtitlesformat,
'subtitleslang': opts.subtitleslang, 'subtitleslangs': opts.subtitleslang,
'matchtitle': decodeOption(opts.matchtitle), 'matchtitle': decodeOption(opts.matchtitle),
'rejecttitle': decodeOption(opts.rejecttitle), 'rejecttitle': decodeOption(opts.rejecttitle),
'max_downloads': opts.max_downloads, 'max_downloads': opts.max_downloads,
@ -608,6 +617,8 @@ def _real_main(argv=None):
ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites)) ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
if opts.recodevideo: if opts.recodevideo:
ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
if opts.embedsubtitles:
ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
# Update version # Update version
if opts.update_self: if opts.update_self:

@ -57,6 +57,7 @@ from .pornotube import PornotubeIE
from .rbmaradio import RBMARadioIE from .rbmaradio import RBMARadioIE
from .redtube import RedTubeIE from .redtube import RedTubeIE
from .ringtv import RingTVIE from .ringtv import RingTVIE
from .ro220 import Ro220IE
from .roxwel import RoxwelIE from .roxwel import RoxwelIE
from .rtlnow import RTLnowIE from .rtlnow import RTLnowIE
from .sina import SinaIE from .sina import SinaIE
@ -116,12 +117,14 @@ _ALL_CLASSES = [
] ]
_ALL_CLASSES.append(GenericIE) _ALL_CLASSES.append(GenericIE)
def gen_extractors(): def gen_extractors():
""" Return a list of an instance of every supported extractor. """ Return a list of an instance of every supported extractor.
The order does matter; the first extractor matched is the one handling the URL. The order does matter; the first extractor matched is the one handling the URL.
""" """
return [klass() for klass in _ALL_CLASSES] return [klass() for klass in _ALL_CLASSES]
def get_info_extractor(ie_name): def get_info_extractor(ie_name):
"""Returns the info extractor class with the given ie_name""" """Returns the info extractor class with the given ie_name"""
return globals()[ie_name+'IE'] return globals()[ie_name+'IE']

@ -47,7 +47,8 @@ class InfoExtractor(object):
uploader_id: Nickname or id of the video uploader. uploader_id: Nickname or id of the video uploader.
location: Physical location of the video. location: Physical location of the video.
player_url: SWF Player URL (used for rtmpdump). player_url: SWF Player URL (used for rtmpdump).
subtitles: The subtitle file contents. subtitles: The subtitle file contents as a dictionary in the format
{language: subtitles}.
view_count: How many users have watched the video on the platform. view_count: How many users have watched the video on the platform.
urlhandle: [internal] The urlHandle to be used to download the file, urlhandle: [internal] The urlHandle to be used to download the file,
like returned by urllib.request.urlopen like returned by urllib.request.urlopen

@ -7,12 +7,14 @@ from .common import InfoExtractor
from ..utils import ( from ..utils import (
compat_urllib_error, compat_urllib_error,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_urlparse,
compat_urllib_request, compat_urllib_request,
ExtractorError, ExtractorError,
) )
from .brightcove import BrightcoveIE from .brightcove import BrightcoveIE
class GenericIE(InfoExtractor): class GenericIE(InfoExtractor):
IE_DESC = u'Generic downloader that works on some sites' IE_DESC = u'Generic downloader that works on some sites'
_VALID_URL = r'.*' _VALID_URL = r'.*'
@ -161,6 +163,10 @@ class GenericIE(InfoExtractor):
raise ExtractorError(u'Invalid URL: %s' % url) raise ExtractorError(u'Invalid URL: %s' % url)
video_url = compat_urllib_parse.unquote(mobj.group(1)) video_url = compat_urllib_parse.unquote(mobj.group(1))
if video_url.startswith('//'):
video_url = compat_urllib_parse_urlparse(url).scheme + ':' + video_url
if '://' not in video_url:
video_url = url + ('' if url.endswith('/') else '/') + video_url
video_id = os.path.basename(video_url) video_id = os.path.basename(video_url)
# here's a fun little line of code for you: # here's a fun little line of code for you:

@ -0,0 +1,42 @@
import re
from .common import InfoExtractor
from ..utils import (
clean_html,
compat_parse_qs,
)
class Ro220IE(InfoExtractor):
IE_NAME = '220.ro'
_VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<video_id>[^/]+)'
_TEST = {
u"url": u"http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/",
u'file': u'LYV6doKo7f.mp4',
u'md5': u'03af18b73a07b4088753930db7a34add',
u'info_dict': {
u"title": u"Luati-le Banii sez 4 ep 1",
u"description": u"Iata-ne reveniti dupa o binemeritata vacanta. Va astept si pe Facebook cu pareri si comentarii.",
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('video_id')
webpage = self._download_webpage(url, video_id)
flashVars_str = self._search_regex(
r'<param name="flashVars" value="([^"]+)"',
webpage, u'flashVars')
flashVars = compat_parse_qs(flashVars_str)
info = {
'_type': 'video',
'id': video_id,
'ext': 'mp4',
'url': flashVars['videoURL'][0],
'title': flashVars['title'][0],
'description': clean_html(flashVars['desc'][0]),
'thumbnail': flashVars['preview'][0],
}
return info

@ -14,7 +14,7 @@ class VideofyMeIE(InfoExtractor):
_TEST = { _TEST = {
u'url': u'http://www.videofy.me/thisisvideofyme/1100701', u'url': u'http://www.videofy.me/thisisvideofyme/1100701',
u'file': u'1100701.mp4', u'file': u'1100701.mp4',
u'md5': u'2046dd5758541d630bfa93e741e2fd79', u'md5': u'c77d700bdc16ae2e9f3c26019bd96143',
u'info_dict': { u'info_dict': {
u'title': u'This is VideofyMe', u'title': u'This is VideofyMe',
u'description': None, u'description': None,
@ -32,9 +32,8 @@ class VideofyMeIE(InfoExtractor):
config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8')) config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8'))
video = config.find('video') video = config.find('video')
sources = video.find('sources') sources = video.find('sources')
url_node = find_xpath_attr(sources, 'source', 'id', 'HQ on') url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key)
if url_node is None: for key in ['on', 'av', 'off']] if node is not None)
url_node = find_xpath_attr(sources, 'source', 'id', 'HQ off')
video_url = url_node.find('url').text video_url = url_node.find('url').text
return {'id': video_id, return {'id': video_id,

@ -3,7 +3,8 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
compat_urllib_parse, compat_urllib_parse,
unescapeHTML,
determine_ext,
ExtractorError, ExtractorError,
) )
@ -36,15 +37,16 @@ class XHamsterIE(InfoExtractor):
video_url = compat_urllib_parse.unquote(mobj.group('file')) video_url = compat_urllib_parse.unquote(mobj.group('file'))
else: else:
video_url = mobj.group('server')+'/key='+mobj.group('file') video_url = mobj.group('server')+'/key='+mobj.group('file')
video_extension = video_url.split('.')[-1]
video_title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>', video_title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>',
webpage, u'title') webpage, u'title')
# Can't see the description anywhere in the UI # Only a few videos have an description
# video_description = self._html_search_regex(r'<span>Description: </span>(?P<description>[^<]+)', mobj = re.search('<span>Description: </span>(?P<description>[^<]+)', webpage)
# webpage, u'description', fatal=False) if mobj:
# if video_description: video_description = unescapeHTML(video_description) video_description = unescapeHTML(mobj.group('description'))
else:
video_description = None
mobj = re.search(r'hint=\'(?P<upload_date_Y>[0-9]{4})-(?P<upload_date_m>[0-9]{2})-(?P<upload_date_d>[0-9]{2}) [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{3,4}\'', webpage) mobj = re.search(r'hint=\'(?P<upload_date_Y>[0-9]{4})-(?P<upload_date_m>[0-9]{2})-(?P<upload_date_d>[0-9]{2}) [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{3,4}\'', webpage)
if mobj: if mobj:
@ -62,9 +64,9 @@ class XHamsterIE(InfoExtractor):
return [{ return [{
'id': video_id, 'id': video_id,
'url': video_url, 'url': video_url,
'ext': video_extension, 'ext': determine_ext(video_url),
'title': video_title, 'title': video_title,
# 'description': video_description, 'description': video_description,
'upload_date': video_upload_date, 'upload_date': video_upload_date,
'uploader_id': video_uploader_id, 'uploader_id': video_uploader_id,
'thumbnail': video_thumbnail 'thumbnail': video_thumbnail

@ -155,11 +155,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Listed in order of quality # Listed in order of quality
_available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13', _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13',
'95', '94', '93', '92', '132', '151', '95', '94', '93', '92', '132', '151',
# 3D
'85', '84', '102', '83', '101', '82', '100', '85', '84', '102', '83', '101', '82', '100',
# Dash video
'138', '137', '248', '136', '247', '135', '246',
'245', '244', '134', '243', '133', '242', '160',
# Dash audio
'141', '172', '140', '171', '139',
] ]
_available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13', _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13',
'95', '94', '93', '92', '132', '151', '95', '94', '93', '92', '132', '151',
'85', '102', '84', '101', '83', '100', '82', '85', '102', '84', '101', '83', '100', '82',
# Dash video
'138', '248', '137', '247', '136', '246', '245',
'244', '135', '243', '134', '242', '133', '160',
# Dash audio
'172', '141', '171', '140', '139',
] ]
_video_extensions = { _video_extensions = {
'13': '3gp', '13': '3gp',
@ -190,6 +201,29 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'96': 'mp4', '96': 'mp4',
'132': 'mp4', '132': 'mp4',
'151': 'mp4', '151': 'mp4',
# Dash mp4
'133': 'mp4',
'134': 'mp4',
'135': 'mp4',
'136': 'mp4',
'137': 'mp4',
'138': 'mp4',
'139': 'mp4',
'140': 'mp4',
'141': 'mp4',
'160': 'mp4',
# Dash webm
'171': 'webm',
'172': 'webm',
'242': 'webm',
'243': 'webm',
'244': 'webm',
'245': 'webm',
'246': 'webm',
'247': 'webm',
'248': 'webm',
} }
_video_dimensions = { _video_dimensions = {
'5': '240x400', '5': '240x400',
@ -220,8 +254,55 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'102': '720p', '102': '720p',
'132': '240p', '132': '240p',
'151': '72p', '151': '72p',
'133': '240p',
'134': '360p',
'135': '480p',
'136': '720p',
'137': '1080p',
'138': '>1080p',
'139': '48k',
'140': '128k',
'141': '256k',
'160': '192p',
'171': '128k',
'172': '256k',
'242': '240p',
'243': '360p',
'244': '480p',
'245': '480p',
'246': '480p',
'247': '720p',
'248': '1080p',
}
_special_itags = {
'82': '3D',
'83': '3D',
'84': '3D',
'85': '3D',
'100': '3D',
'101': '3D',
'102': '3D',
'133': 'DASH Video',
'134': 'DASH Video',
'135': 'DASH Video',
'136': 'DASH Video',
'137': 'DASH Video',
'138': 'DASH Video',
'139': 'DASH Audio',
'140': 'DASH Audio',
'141': 'DASH Audio',
'160': 'DASH Video',
'171': 'DASH Audio',
'172': 'DASH Audio',
'242': 'DASH Video',
'243': 'DASH Video',
'244': 'DASH Video',
'245': 'DASH Video',
'246': 'DASH Video',
'247': 'DASH Video',
'248': 'DASH Video',
} }
_3d_itags = ['85', '84', '102', '83', '101', '82', '100']
IE_NAME = u'youtube' IE_NAME = u'youtube'
_TESTS = [ _TESTS = [
{ {
@ -342,17 +423,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
elif len(s) == 87: elif len(s) == 87:
return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:] return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
elif len(s) == 86: elif len(s) == 86:
return s[5:20] + s[2] + s[21:] return s[5:40] + s[3] + s[41:48] + s[0] + s[49:86]
elif len(s) == 85: elif len(s) == 85:
return s[83:34:-1] + s[0] + s[33:27:-1] + s[3] + s[26:19:-1] + s[34] + s[18:3:-1] + s[27] return s[83:34:-1] + s[0] + s[33:27:-1] + s[3] + s[26:19:-1] + s[34] + s[18:3:-1] + s[27]
elif len(s) == 84: elif len(s) == 84:
return s[83:27:-1] + s[0] + s[26:5:-1] + s[2:0:-1] + s[27] return s[5:40] + s[3] + s[41:48] + s[0] + s[49:84]
elif len(s) == 83: elif len(s) == 83:
return s[81:64:-1] + s[82] + s[63:52:-1] + s[45] + s[51:45:-1] + s[1] + s[44:1:-1] + s[0] return s[81:64:-1] + s[82] + s[63:52:-1] + s[45] + s[51:45:-1] + s[1] + s[44:1:-1] + s[0]
elif len(s) == 82: elif len(s) == 82:
return s[36] + s[79:67:-1] + s[81] + s[66:40:-1] + s[33] + s[39:36:-1] + s[40] + s[35] + s[0] + s[67] + s[32:0:-1] + s[34] return s[1:19] + s[0] + s[20:68] + s[19] + s[69:82]
elif len(s) == 81: elif len(s) == 81:
return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9] return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
elif len(s) == 80:
return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
elif len(s) == 79: elif len(s) == 79:
return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9] return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
@ -375,11 +458,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
try: try:
sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8') sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
return (u'unable to download video subtitles: %s' % compat_str(err), None) self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
return {}
sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list) sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list) sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list)
if not sub_lang_list: if not sub_lang_list:
return (u'video doesn\'t have subtitles', None) self._downloader.report_warning(u'video doesn\'t have subtitles')
return {}
return sub_lang_list return sub_lang_list
def _list_available_subtitles(self, video_id): def _list_available_subtitles(self, video_id):
@ -388,8 +473,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
def _request_subtitle(self, sub_lang, sub_name, video_id, format): def _request_subtitle(self, sub_lang, sub_name, video_id, format):
""" """
Return tuple: Return the subtitle as a string or None if they are not found
(error_message, sub_lang, sub)
""" """
self.report_video_subtitles_request(video_id, sub_lang, format) self.report_video_subtitles_request(video_id, sub_lang, format)
params = compat_urllib_parse.urlencode({ params = compat_urllib_parse.urlencode({
@ -402,21 +486,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
try: try:
sub = compat_urllib_request.urlopen(url).read().decode('utf-8') sub = compat_urllib_request.urlopen(url).read().decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
return (u'unable to download video subtitles: %s' % compat_str(err), None, None) self._downloader.report_warning(u'unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
return
if not sub: if not sub:
return (u'Did not fetch video subtitles', None, None) self._downloader.report_warning(u'Did not fetch video subtitles')
return (None, sub_lang, sub) return
return sub
def _request_automatic_caption(self, video_id, webpage): def _request_automatic_caption(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an """We need the webpage for getting the captions url, pass it as an
argument to speed up the process.""" argument to speed up the process."""
sub_lang = self._downloader.params.get('subtitleslang') or 'en' sub_lang = (self._downloader.params.get('subtitleslangs') or ['en'])[0]
sub_format = self._downloader.params.get('subtitlesformat') sub_format = self._downloader.params.get('subtitlesformat')
self.to_screen(u'%s: Looking for automatic captions' % video_id) self.to_screen(u'%s: Looking for automatic captions' % video_id)
mobj = re.search(r';ytplayer.config = ({.*?});', webpage) mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang
if mobj is None: if mobj is None:
return [(err_msg, None, None)] self._downloader.report_warning(err_msg)
return {}
player_config = json.loads(mobj.group(1)) player_config = json.loads(mobj.group(1))
try: try:
args = player_config[u'args'] args = player_config[u'args']
@ -431,40 +518,43 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
}) })
subtitles_url = caption_url + '&' + params subtitles_url = caption_url + '&' + params
sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions') sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions')
return [(None, sub_lang, sub)] return {sub_lang: sub}
except KeyError: # An extractor error can be raise by the download process if there are
return [(err_msg, None, None)] # no automatic captions but there are subtitles
except (KeyError, ExtractorError):
def _extract_subtitle(self, video_id): self._downloader.report_warning(err_msg)
return {}
def _extract_subtitles(self, video_id):
""" """
Return a list with a tuple: Return a dictionary: {language: subtitles} or {} if the subtitles
[(error_message, sub_lang, sub)] couldn't be found
""" """
sub_lang_list = self._get_available_subtitles(video_id) available_subs_list = self._get_available_subtitles(video_id)
sub_format = self._downloader.params.get('subtitlesformat') sub_format = self._downloader.params.get('subtitlesformat')
if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles if not available_subs_list: #There was some error, it didn't get the available subtitles
return [(sub_lang_list[0], None, None)] return {}
if self._downloader.params.get('subtitleslang', False): if self._downloader.params.get('allsubtitles', False):
sub_lang = self._downloader.params.get('subtitleslang') sub_lang_list = available_subs_list
elif 'en' in sub_lang_list:
sub_lang = 'en'
else: else:
sub_lang = list(sub_lang_list.keys())[0] if self._downloader.params.get('subtitleslangs', False):
if not sub_lang in sub_lang_list: reqested_langs = self._downloader.params.get('subtitleslangs')
return [(u'no closed captions found in the specified language "%s"' % sub_lang, None, None)] elif 'en' in available_subs_list:
reqested_langs = ['en']
subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) else:
return [subtitle] reqested_langs = [list(available_subs_list.keys())[0]]
def _extract_all_subtitles(self, video_id): sub_lang_list = {}
sub_lang_list = self._get_available_subtitles(video_id) for sub_lang in reqested_langs:
sub_format = self._downloader.params.get('subtitlesformat') if not sub_lang in available_subs_list:
if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
return [(sub_lang_list[0], None, None)] continue
subtitles = [] sub_lang_list[sub_lang] = available_subs_list[sub_lang]
subtitles = {}
for sub_lang in sub_lang_list: for sub_lang in sub_lang_list:
subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
subtitles.append(subtitle) if subtitle:
subtitles[sub_lang] = subtitle
return subtitles return subtitles
def _print_formats(self, formats): def _print_formats(self, formats):
@ -472,7 +562,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
for x in formats: for x in formats:
print('%s\t:\t%s\t[%s]%s' %(x, self._video_extensions.get(x, 'flv'), print('%s\t:\t%s\t[%s]%s' %(x, self._video_extensions.get(x, 'flv'),
self._video_dimensions.get(x, '???'), self._video_dimensions.get(x, '???'),
' (3D)' if x in self._3d_itags else '')) ' ('+self._special_itags[x]+')' if x in self._special_itags else ''))
def _extract_id(self, url): def _extract_id(self, url):
mobj = re.match(self._VALID_URL, url, re.VERBOSE) mobj = re.match(self._VALID_URL, url, re.VERBOSE)
@ -655,25 +745,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# subtitles # subtitles
video_subtitles = None video_subtitles = None
if self._downloader.params.get('writesubtitles', False): if self._downloader.params.get('writesubtitles', False) or self._downloader.params.get('allsubtitles', False):
video_subtitles = self._extract_subtitle(video_id) video_subtitles = self._extract_subtitles(video_id)
if video_subtitles: elif self._downloader.params.get('writeautomaticsub', False):
(sub_error, sub_lang, sub) = video_subtitles[0]
if sub_error:
self._downloader.report_warning(sub_error)
if self._downloader.params.get('writeautomaticsub', False):
video_subtitles = self._request_automatic_caption(video_id, video_webpage) video_subtitles = self._request_automatic_caption(video_id, video_webpage)
(sub_error, sub_lang, sub) = video_subtitles[0]
if sub_error:
self._downloader.report_warning(sub_error)
if self._downloader.params.get('allsubtitles', False):
video_subtitles = self._extract_all_subtitles(video_id)
for video_subtitle in video_subtitles:
(sub_error, sub_lang, sub) = video_subtitle
if sub_error:
self._downloader.report_warning(sub_error)
if self._downloader.params.get('listsubtitles', False): if self._downloader.params.get('listsubtitles', False):
self._list_available_subtitles(video_id) self._list_available_subtitles(video_id)
@ -699,6 +774,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if m_s is not None: if m_s is not None:
self.to_screen(u'%s: Encrypted signatures detected.' % video_id) self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']] video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
m_s = re.search(r'[&,]s=', args.get('adaptive_fmts', u''))
if m_s is not None:
if 'url_encoded_fmt_stream_map' in video_info:
video_info['url_encoded_fmt_stream_map'][0] += ',' + args['adaptive_fmts']
else:
video_info['url_encoded_fmt_stream_map'] = [args['adaptive_fmts']]
elif 'adaptive_fmts' in video_info:
if 'url_encoded_fmt_stream_map' in video_info:
video_info['url_encoded_fmt_stream_map'][0] += ',' + video_info['adaptive_fmts'][0]
else:
video_info['url_encoded_fmt_stream_map'] = video_info['adaptive_fmts']
except ValueError: except ValueError:
pass pass
@ -758,7 +844,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
video_format = '{0} - {1}{2}'.format(format_param if format_param else video_extension, video_format = '{0} - {1}{2}'.format(format_param if format_param else video_extension,
self._video_dimensions.get(format_param, '???'), self._video_dimensions.get(format_param, '???'),
' (3D)' if format_param in self._3d_itags else '') ' ('+self._special_itags[format_param]+')' if format_param in self._special_itags else '')
results.append({ results.append({
'id': video_id, 'id': video_id,

@ -476,7 +476,7 @@ def formatSeconds(secs):
def make_HTTPS_handler(opts): def make_HTTPS_handler(opts):
if sys.version_info < (3,2): if sys.version_info < (3,2):
# Python's 2.x handler is very simplistic # Python's 2.x handler is very simplistic
return compat_urllib_request.HTTPSHandler() return YoutubeDLHandlerHTTPS()
else: else:
import ssl import ssl
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@ -485,7 +485,7 @@ def make_HTTPS_handler(opts):
context.verify_mode = (ssl.CERT_NONE context.verify_mode = (ssl.CERT_NONE
if opts.no_check_certificate if opts.no_check_certificate
else ssl.CERT_REQUIRED) else ssl.CERT_REQUIRED)
return compat_urllib_request.HTTPSHandler(context=context) return YoutubeDLHandlerHTTPS(context=context)
class ExtractorError(Exception): class ExtractorError(Exception):
"""Error during info extraction.""" """Error during info extraction."""
@ -569,7 +569,8 @@ class ContentTooShortError(Exception):
self.downloaded = downloaded self.downloaded = downloaded
self.expected = expected self.expected = expected
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
class YoutubeDLHandler_Template: # Old-style class, like HTTPHandler
"""Handler for HTTP requests and responses. """Handler for HTTP requests and responses.
This class, when installed with an OpenerDirector, automatically adds This class, when installed with an OpenerDirector, automatically adds
@ -602,8 +603,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
ret.code = code ret.code = code
return ret return ret
def http_request(self, req): def _http_request(self, req):
for h,v in std_headers.items(): for h, v in std_headers.items():
if h in req.headers: if h in req.headers:
del req.headers[h] del req.headers[h]
req.add_header(h, v) req.add_header(h, v)
@ -618,7 +619,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
del req.headers['Youtubedl-user-agent'] del req.headers['Youtubedl-user-agent']
return req return req
def http_response(self, req, resp): def _http_response(self, req, resp):
old_resp = resp old_resp = resp
# gzip # gzip
if resp.headers.get('Content-encoding', '') == 'gzip': if resp.headers.get('Content-encoding', '') == 'gzip':
@ -632,8 +633,16 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
resp.msg = old_resp.msg resp.msg = old_resp.msg
return resp return resp
https_request = http_request
https_response = http_response class YoutubeDLHandler(YoutubeDLHandler_Template, compat_urllib_request.HTTPHandler):
http_request = YoutubeDLHandler_Template._http_request
http_response = YoutubeDLHandler_Template._http_response
class YoutubeDLHandlerHTTPS(YoutubeDLHandler_Template, compat_urllib_request.HTTPSHandler):
https_request = YoutubeDLHandler_Template._http_request
https_response = YoutubeDLHandler_Template._http_response
def unified_strdate(date_str): def unified_strdate(date_str):
"""Return a string with the date in the format YYYYMMDD""" """Return a string with the date in the format YYYYMMDD"""
@ -657,6 +666,9 @@ def determine_ext(url, default_ext=u'unknown_video'):
else: else:
return default_ext return default_ext
def subtitles_filename(filename, sub_lang, sub_format):
return filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
def date_from_str(date_str): def date_from_str(date_str):
""" """
Return a datetime object from a string in the format YYYYMMDD or Return a datetime object from a string in the format YYYYMMDD or

@ -1,2 +1,2 @@
__version__ = '2013.08.21' __version__ = '2013.08.23'

Loading…
Cancel
Save