@ -74,13 +74,13 @@ from .utils import (
ExtractorError ,
ExtractorError ,
GeoRestrictedError ,
GeoRestrictedError ,
HEADRequest ,
HEADRequest ,
InAdvancePagedList ,
ISO3166Utils ,
ISO3166Utils ,
LazyList ,
LazyList ,
MaxDownloadsReached ,
MaxDownloadsReached ,
Namespace ,
Namespace ,
PagedList ,
PagedList ,
PerRequestProxyHandler ,
PerRequestProxyHandler ,
PlaylistEntries ,
Popen ,
Popen ,
PostProcessingError ,
PostProcessingError ,
ReExtractInfo ,
ReExtractInfo ,
@ -1410,7 +1410,7 @@ class YoutubeDL:
else :
else :
self . report_error ( ' no suitable InfoExtractor for URL %s ' % url )
self . report_error ( ' no suitable InfoExtractor for URL %s ' % url )
def _ _ handle_extraction_exceptions( func ) :
def _ handle_extraction_exceptions( func ) :
@functools.wraps ( func )
@functools.wraps ( func )
def wrapper ( self , * args , * * kwargs ) :
def wrapper ( self , * args , * * kwargs ) :
while True :
while True :
@ -1483,7 +1483,7 @@ class YoutubeDL:
self . to_screen ( ' ' )
self . to_screen ( ' ' )
raise
raise
@_ _ handle_extraction_exceptions
@_ handle_extraction_exceptions
def __extract_info ( self , url , ie , download , extra_info , process ) :
def __extract_info ( self , url , ie , download , extra_info , process ) :
ie_result = ie . extract ( url )
ie_result = ie . extract ( url )
if ie_result is None : # Finished already (backwards compatibility; listformats and friends should be moved here)
if ie_result is None : # Finished already (backwards compatibility; listformats and friends should be moved here)
@ -1666,105 +1666,14 @@ class YoutubeDL:
}
}
def __process_playlist ( self , ie_result , download ) :
def __process_playlist ( self , ie_result , download ) :
# We process each entry in the playlist
""" Process each entry in the playlist """
playlist = ie_result . get ( ' title ' ) or ie_result . get ( ' id ' )
title = ie_result . get ( ' title ' ) or ie_result . get ( ' id ' ) or ' <Untitled> '
self . to_screen ( ' [download] Downloading playlist: %s ' % playlist )
self . to_screen ( f ' [download] Downloading playlist: { title } ' )
if ' entries ' not in ie_result :
raise EntryNotInPlaylist ( ' There are no entries ' )
MissingEntry = object ( )
incomplete_entries = bool ( ie_result . get ( ' requested_entries ' ) )
if incomplete_entries :
def fill_missing_entries ( entries , indices ) :
ret = [ MissingEntry ] * max ( indices )
for i , entry in zip ( indices , entries ) :
ret [ i - 1 ] = entry
return ret
ie_result [ ' entries ' ] = fill_missing_entries ( ie_result [ ' entries ' ] , ie_result [ ' requested_entries ' ] )
playlist_results = [ ]
playliststart = self . params . get ( ' playliststart ' , 1 )
playlistend = self . params . get ( ' playlistend ' )
# For backwards compatibility, interpret -1 as whole list
if playlistend == - 1 :
playlistend = None
playlistitems_str = self . params . get ( ' playlist_items ' )
playlistitems = None
if playlistitems_str is not None :
def iter_playlistitems ( format ) :
for string_segment in format . split ( ' , ' ) :
if ' - ' in string_segment :
start , end = string_segment . split ( ' - ' )
for item in range ( int ( start ) , int ( end ) + 1 ) :
yield int ( item )
else :
yield int ( string_segment )
playlistitems = orderedSet ( iter_playlistitems ( playlistitems_str ) )
ie_entries = ie_result [ ' entries ' ]
if isinstance ( ie_entries , list ) :
playlist_count = len ( ie_entries )
msg = f ' Collected { playlist_count } videos; downloading %d of them '
ie_result [ ' playlist_count ' ] = ie_result . get ( ' playlist_count ' ) or playlist_count
def get_entry ( i ) :
return ie_entries [ i - 1 ]
else :
msg = ' Downloading %d videos '
if not isinstance ( ie_entries , ( PagedList , LazyList ) ) :
ie_entries = LazyList ( ie_entries )
elif isinstance ( ie_entries , InAdvancePagedList ) :
if ie_entries . _pagesize == 1 :
playlist_count = ie_entries . _pagecount
def get_entry ( i ) :
return YoutubeDL . __handle_extraction_exceptions (
lambda self , i : ie_entries [ i - 1 ]
) ( self , i )
entries , broken = [ ] , False
items = playlistitems if playlistitems is not None else itertools . count ( playliststart )
for i in items :
if i == 0 :
continue
if playlistitems is None and playlistend is not None and playlistend < i :
break
entry = None
try :
entry = get_entry ( i )
if entry is MissingEntry :
raise EntryNotInPlaylist ( )
except ( IndexError , EntryNotInPlaylist ) :
if incomplete_entries :
raise EntryNotInPlaylist ( f ' Entry { i } cannot be found ' )
elif not playlistitems :
break
entries . append ( entry )
try :
if entry is not None :
# TODO: Add auto-generated fields
self . _match_entry ( entry , incomplete = True , silent = True )
except ( ExistingVideoReached , RejectedVideoReached ) :
broken = True
break
ie_result [ ' entries ' ] = entries
# Save playlist_index before re-ordering
entries = [
( ( playlistitems [ i - 1 ] if playlistitems else i + playliststart - 1 ) , entry )
for i , entry in enumerate ( entries , 1 )
if entry is not None ]
n_entries = len ( entries )
if not ( ie_result . get ( ' playlist_count ' ) or broken or playlistitems or playlistend ) :
all_entries = PlaylistEntries ( self , ie_result )
ie_result [ ' playlist_count ' ] = n_entries
entries = orderedSet ( all_entries . get_requested_items ( ) )
ie_result [ ' requested_entries ' ] , ie_result [ ' entries ' ] = tuple ( zip ( * entries ) ) or ( [ ] , [ ] )
if not playlistitems and ( playliststart != 1 or playlistend ) :
n_entries , ie_result [ ' playlist_count ' ] = len ( entries ) , all_entries . full_count
playlistitems = list ( range ( playliststart , playliststart + n_entries ) )
ie_result [ ' requested_entries ' ] = playlistitems
_infojson_written = False
_infojson_written = False
write_playlist_files = self . params . get ( ' allow_playlist_files ' , True )
write_playlist_files = self . params . get ( ' allow_playlist_files ' , True )
@ -1787,28 +1696,29 @@ class YoutubeDL:
if self . params . get ( ' playlistrandom ' , False ) :
if self . params . get ( ' playlistrandom ' , False ) :
random . shuffle ( entries )
random . shuffle ( entries )
x_forwarded_for = ie_result . get ( ' __x_forwarded_for_ip ' )
self . to_screen ( f ' [ { ie_result [ " extractor " ] } ] Playlist { title } : Downloading { n_entries } videos '
f ' { format_field ( ie_result , " playlist_count " , " of %s " ) } ' )
self . to_screen ( f ' [ { ie_result [ " extractor " ] } ] playlist { playlist } : { msg % n_entries } ' )
failures = 0
failures = 0
max_failures = self . params . get ( ' skip_playlist_after_errors ' ) or float ( ' inf ' )
max_failures = self . params . get ( ' skip_playlist_after_errors ' ) or float ( ' inf ' )
for i , entry_tuple in enumerate ( entries , 1 ) :
for i , ( playlist_index , entry ) in enumerate ( entries , 1 ) :
playlist_index , entry = entry_tuple
# TODO: Add auto-generated fields
if ' playlist-index ' in self . params [ ' compat_opts ' ] :
if self . _match_entry ( entry , incomplete = True ) is not None :
playlist_index = playlistitems [ i - 1 ] if playlistitems else i + playliststart - 1
continue
if ' playlist-index ' in self . params . get ( ' compat_opts ' , [ ] ) :
playlist_index = ie_result [ ' requested_entries ' ] [ i - 1 ]
self . to_screen ( ' [download] Downloading video %s of %s ' % (
self . to_screen ( ' [download] Downloading video %s of %s ' % (
self . _format_screen ( i , self . Styles . ID ) , self . _format_screen ( n_entries , self . Styles . EMPHASIS ) ) )
self . _format_screen ( i , self . Styles . ID ) , self . _format_screen ( n_entries , self . Styles . EMPHASIS ) ) )
# This __x_forwarded_for_ip thing is a bit ugly but requires
# minimal changes
entry [ ' __x_forwarded_for_ip ' ] = ie_result . get ( ' __x_forwarded_for_ip ' )
if x_forwarded_for :
entry_result = self . __process_iterable_entry ( entry , download , {
entry [ ' __x_forwarded_for_ip ' ] = x_forwarded_for
extra = {
' n_entries ' : n_entries ,
' n_entries ' : n_entries ,
' __last_playlist_index ' : max ( playlistitems) if playlistitems else ( playlistend or n_entries ) ,
' __last_playlist_index ' : max ( ie_result [ ' requested_entries ' ] ) ,
' playlist_count ' : ie_result . get ( ' playlist_count ' ) ,
' playlist_count ' : ie_result . get ( ' playlist_count ' ) ,
' playlist_index ' : playlist_index ,
' playlist_index ' : playlist_index ,
' playlist_autonumber ' : i ,
' playlist_autonumber ' : i ,
' playlist ' : playlist ,
' playlist ' : title ,
' playlist_id ' : ie_result . get ( ' id ' ) ,
' playlist_id ' : ie_result . get ( ' id ' ) ,
' playlist_title ' : ie_result . get ( ' title ' ) ,
' playlist_title ' : ie_result . get ( ' title ' ) ,
' playlist_uploader ' : ie_result . get ( ' uploader ' ) ,
' playlist_uploader ' : ie_result . get ( ' uploader ' ) ,
@ -1818,20 +1728,17 @@ class YoutubeDL:
' webpage_url_basename ' : url_basename ( ie_result [ ' webpage_url ' ] ) ,
' webpage_url_basename ' : url_basename ( ie_result [ ' webpage_url ' ] ) ,
' webpage_url_domain ' : get_domain ( ie_result [ ' webpage_url ' ] ) ,
' webpage_url_domain ' : get_domain ( ie_result [ ' webpage_url ' ] ) ,
' extractor_key ' : ie_result [ ' extractor_key ' ] ,
' extractor_key ' : ie_result [ ' extractor_key ' ] ,
}
} )
if self . _match_entry ( entry , incomplete = True ) is not None :
continue
entry_result = self . __process_iterable_entry ( entry , download , extra )
if not entry_result :
if not entry_result :
failures + = 1
failures + = 1
if failures > = max_failures :
if failures > = max_failures :
self . report_error (
self . report_error (
' Skipping the remaining entries in playlist " %s" since %d items failed extraction ' % ( playlist , failures ) )
f ' Skipping the remaining entries in playlist " { title } " since { failures } items failed extraction ' )
break
break
playlist_results . append ( entry_result )
entries [ i - 1 ] = ( playlist_index , entry_result )
ie_result [ ' entries ' ] = playlist_results
# Update with processed data
ie_result [ ' requested_entries ' ] , ie_result [ ' entries ' ] = tuple ( zip ( * entries ) ) or ( [ ] , [ ] )
# Write the updated info to json
# Write the updated info to json
if _infojson_written is True and self . _write_info_json (
if _infojson_written is True and self . _write_info_json (
@ -1840,10 +1747,10 @@ class YoutubeDL:
return
return
ie_result = self . run_all_pps ( ' playlist ' , ie_result )
ie_result = self . run_all_pps ( ' playlist ' , ie_result )
self . to_screen ( f ' [download] Finished downloading playlist: { playlist } ' )
self . to_screen ( f ' [download] Finished downloading playlist: { title } ' )
return ie_result
return ie_result
@_ _ handle_extraction_exceptions
@_ handle_extraction_exceptions
def __process_iterable_entry ( self , entry , download , extra_info ) :
def __process_iterable_entry ( self , entry , download , extra_info ) :
return self . process_ie_result (
return self . process_ie_result (
entry , download = download , extra_info = extra_info )
entry , download = download , extra_info = extra_info )
pukkandan
2 years ago