From 29f7c58aafb25a094e267a8a3fb355e102e42792 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 1 Jan 2021 17:56:37 +0530 Subject: [PATCH] Update to ytdl-2021.01.03 --- README.md | 3 +- docs/supportedsites.md | 54 +- test/test_InfoExtractor.py | 61 ++ test/test_all_urls.py | 6 +- test/test_utils.py | 5 + youtube_dlc/YoutubeDL.py | 6 +- youtube_dlc/downloader/hls.py | 2 + youtube_dlc/extractor/acast.py | 116 ++- youtube_dlc/extractor/aenetworks.py | 339 +++++--- youtube_dlc/extractor/amcnetworks.py | 51 +- youtube_dlc/extractor/americastestkitchen.py | 68 +- youtube_dlc/extractor/anvato.py | 89 +- .../anvato_token_generator/__init__.py | 7 + .../anvato_token_generator/common.py | 6 + .../extractor/anvato_token_generator/nfl.py | 30 + youtube_dlc/extractor/aparat.py | 20 +- youtube_dlc/extractor/arcpublishing.py | 174 ++++ youtube_dlc/extractor/arkena.py | 152 ++-- youtube_dlc/extractor/asiancrush.py | 221 +++-- youtube_dlc/extractor/bbc.py | 74 +- youtube_dlc/extractor/beampro.py | 194 ----- youtube_dlc/extractor/bongacams.py | 60 ++ youtube_dlc/extractor/brightcove.py | 73 +- youtube_dlc/extractor/cbslocal.py | 67 +- youtube_dlc/extractor/cnn.py | 5 +- youtube_dlc/extractor/common.py | 56 +- youtube_dlc/extractor/cspan.py | 23 + youtube_dlc/extractor/ctv.py | 52 ++ youtube_dlc/extractor/drtv.py | 5 +- youtube_dlc/extractor/eporner.py | 13 +- youtube_dlc/extractor/extractors.py | 85 +- youtube_dlc/extractor/facebook.py | 300 +++++-- youtube_dlc/extractor/fujitv.py | 35 + youtube_dlc/extractor/gamespot.py | 110 +-- youtube_dlc/extractor/generic.py | 179 ++-- youtube_dlc/extractor/go.py | 21 +- youtube_dlc/extractor/instagram.py | 154 ++-- youtube_dlc/extractor/itv.py | 307 ++----- youtube_dlc/extractor/lbry.py | 197 ++++- youtube_dlc/extractor/linuxacademy.py | 130 ++- youtube_dlc/extractor/mdr.py | 77 +- youtube_dlc/extractor/mediaset.py | 5 +- youtube_dlc/extractor/mitele.py | 48 +- youtube_dlc/extractor/nba.py | 480 ++++++++--- youtube_dlc/extractor/nbc.py | 53 +- youtube_dlc/extractor/nfl.py | 257 ++---- youtube_dlc/extractor/nhk.py | 179 +++- youtube_dlc/extractor/niconico.py | 97 ++- youtube_dlc/extractor/ninecninemedia.py | 16 +- youtube_dlc/extractor/nrk.py | 812 ++++++++++-------- youtube_dlc/extractor/peertube.py | 4 + youtube_dlc/extractor/piksel.py | 109 ++- youtube_dlc/extractor/pornhub.py | 46 +- youtube_dlc/extractor/reddit.py | 35 +- youtube_dlc/extractor/ruutu.py | 92 +- youtube_dlc/extractor/sevenplus.py | 32 +- youtube_dlc/extractor/sky.py | 113 ++- youtube_dlc/extractor/slideslive.py | 56 +- youtube_dlc/extractor/smotri.py | 416 --------- youtube_dlc/extractor/sonyliv.py | 112 ++- youtube_dlc/extractor/spankbang.py | 36 +- youtube_dlc/extractor/sprout.py | 88 +- youtube_dlc/extractor/stitcher.py | 60 +- youtube_dlc/extractor/streetvoice.py | 93 +- youtube_dlc/extractor/teachable.py | 4 +- youtube_dlc/extractor/telecinco.py | 77 +- youtube_dlc/extractor/telequebec.py | 160 ++-- youtube_dlc/extractor/tenplay.py | 34 +- youtube_dlc/extractor/theplatform.py | 3 + youtube_dlc/extractor/theweatherchannel.py | 43 +- youtube_dlc/extractor/toggle.py | 107 ++- youtube_dlc/extractor/tubitv.py | 14 + youtube_dlc/extractor/turner.py | 44 +- youtube_dlc/extractor/tv5unis.py | 121 +++ youtube_dlc/extractor/tva.py | 65 +- youtube_dlc/extractor/tver.py | 67 ++ youtube_dlc/extractor/tvplay.py | 90 +- youtube_dlc/extractor/twitcasting.py | 72 +- youtube_dlc/extractor/uktvplay.py | 11 +- youtube_dlc/extractor/videa.py | 99 ++- youtube_dlc/extractor/videomore.py | 255 +++--- youtube_dlc/extractor/viki.py | 6 +- youtube_dlc/extractor/vimeo.py | 7 + youtube_dlc/extractor/vlive.py | 1 + youtube_dlc/extractor/vvvvid.py | 125 ++- youtube_dlc/extractor/washingtonpost.py | 101 +-- youtube_dlc/extractor/wdr.py | 37 +- youtube_dlc/extractor/wistia.py | 159 ++-- youtube_dlc/extractor/yandexdisk.py | 149 ++-- youtube_dlc/extractor/yandexmusic.py | 307 +++++-- youtube_dlc/extractor/yandexvideo.py | 122 ++- youtube_dlc/extractor/youtube.py | 294 +++++-- youtube_dlc/extractor/zdf.py | 4 +- youtube_dlc/extractor/zype.py | 8 +- youtube_dlc/options.py | 2 +- youtube_dlc/utils.py | 2 +- 96 files changed, 5721 insertions(+), 3735 deletions(-) create mode 100644 youtube_dlc/extractor/anvato_token_generator/__init__.py create mode 100644 youtube_dlc/extractor/anvato_token_generator/common.py create mode 100644 youtube_dlc/extractor/anvato_token_generator/nfl.py create mode 100644 youtube_dlc/extractor/arcpublishing.py delete mode 100644 youtube_dlc/extractor/beampro.py create mode 100644 youtube_dlc/extractor/bongacams.py create mode 100644 youtube_dlc/extractor/ctv.py create mode 100644 youtube_dlc/extractor/fujitv.py delete mode 100644 youtube_dlc/extractor/smotri.py create mode 100644 youtube_dlc/extractor/tv5unis.py create mode 100644 youtube_dlc/extractor/tver.py diff --git a/README.md b/README.md index ab1c0547b..681157f6d 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ I will add some memorable short links to the binaries so you can download them e out, youtube-dlc will ask interactively. -2, --twofactor TWOFACTOR Two-factor authentication code -n, --netrc Use .netrc authentication data - --video-password PASSWORD Video password (vimeo, smotri, youku) + --video-password PASSWORD Video password (vimeo, youku) ## Adobe Pass Options: --ap-mso MSO Adobe Pass multiple-system operator (TV @@ -846,6 +846,7 @@ Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends - `container`: Name of the container format - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`) - `format_id`: A short description of the format + - `language`: Language code Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain). diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 0b183b272..8aede26a9 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -34,6 +34,8 @@ - **adobetv:video** - **AdultSwim** - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault + - **aenetworks:collection** + - **aenetworks:show** - **afreecatv**: afreecatv.com - **AirMozilla** - **AliExpressLive** @@ -55,6 +57,7 @@ - **appletrailers** - **appletrailers:section** - **archive.org**: archive.org videos + - **ArcPublishing** - **ARD** - **ARD:mediathek** - **ARDBetaMediathek** @@ -101,6 +104,7 @@ - **BilibiliAudioAlbum** - **BiliBiliPlayer** - **BioBioChileTV** + - **Biography** - **BIQLE** - **BitChute** - **BitChuteChannel** @@ -110,6 +114,7 @@ - **blinkx** - **Bloomberg** - **BokeCC** + - **BongaCams** - **BostonGlobe** - **Box** - **Bpb**: Bundeszentrale für politische Bildung @@ -144,6 +149,7 @@ - **CBS** - **CBSInteractive** - **CBSLocal** + - **CBSLocalArticle** - **cbsnews**: CBS News - **cbsnews:embed** - **cbsnews:livevideo**: CBS News Live Videos @@ -193,9 +199,9 @@ - **CrooksAndLiars** - **crunchyroll** - **crunchyroll:playlist** - - **CSNNE** - **CSpan**: C-SPAN - **CtsNews**: 華視新聞 + - **CTV** - **CTVNews** - **cu.ntv.co.jp**: Nippon Television Network - **Culturebox** @@ -271,7 +277,6 @@ - **ESPNArticle** - **EsriVideo** - **Europa** - - **EveryonesMixtape** - **EWETV** - **ExpoTV** - **Expressen** @@ -313,11 +318,11 @@ - **FrontendMasters** - **FrontendMastersCourse** - **FrontendMastersLesson** + - **FujiTVFODPlus7** - **Funimation** - **Funk** - **Fusion** - **Fux** - - **FXNetworks** - **Gaia** - **GameInformer** - **GameSpot** @@ -350,6 +355,7 @@ - **hgtv.com:show** - **HiDive** - **HistoricFilms** + - **history:player** - **history:topic**: History.com Topic - **hitbox** - **hitbox:live** @@ -403,7 +409,6 @@ - **JWPlatform** - **Kakao** - **Kaltura** - - **KanalPlay**: Kanal 5/9/11 Play - **Kankan** - **Karaoketv** - **KarriereVideos** @@ -427,7 +432,8 @@ - **la7.it** - **laola1tv** - **laola1tv:embed** - - **lbry.tv** + - **lbry** + - **lbry:channel** - **LCI** - **Lcp** - **LcpPlay** @@ -493,6 +499,7 @@ - **META** - **metacafe** - **Metacritic** + - **mewatch** - **Mgoon** - **MGTV**: 芒果TV - **MiaoPai** @@ -503,8 +510,6 @@ - **mixcloud** - **mixcloud:playlist** - **mixcloud:user** - - **Mixer:live** - - **Mixer:vod** - **MLB** - **Mnet** - **MNetTV** @@ -547,6 +552,11 @@ - **Naver** - **Naver:live** - **NBA** + - **nba:watch** + - **nba:watch:collection** + - **NBAChannel** + - **NBAEmbed** + - **NBAWatchEmbed** - **NBC** - **NBCNews** - **nbcolympics** @@ -576,8 +586,10 @@ - **NextTV**: 壹電視 - **Nexx** - **NexxEmbed** - - **nfl.com** + - **nfl.com** (Currently broken) + - **nfl.com:article** (Currently broken) - **NhkVod** + - **NhkVodProgram** - **nhl.com** - **nick.com** - **nick.de** @@ -592,7 +604,6 @@ - **njoy:embed** - **NJPWWorld**: 新日本プロレスワールド - **NobelPrize** - - **Noco** - **NonkTube** - **Noovo** - **Normalboots** @@ -610,6 +621,7 @@ - **Npr** - **NRK** - **NRKPlaylist** + - **NRKRadioPodkast** - **NRKSkole**: NRK Skole - **NRKTV**: NRK TV and NRK Radio - **NRKTVDirekte**: NRK TV Direkte and NRK Radio Direkte @@ -719,6 +731,7 @@ - **qqmusic:singer**: QQ音乐 - 歌手 - **qqmusic:toplist**: QQ音乐 - 排行榜 - **QuantumTV** + - **Qub** - **Quickline** - **QuicklineLive** - **R7** @@ -811,18 +824,17 @@ - **Shared**: shared.sx - **ShowRoomLive** - **Sina** + - **sky.it** + - **sky:news** + - **sky:sports** + - **sky:sports:news** + - **skyacademy.it** - **SkylineWebcams** - - **SkyNews** - **skynewsarabia:article** - **skynewsarabia:video** - - **SkySports** - **Slideshare** - **SlidesLive** - **Slutload** - - **smotri**: Smotri.com - - **smotri:broadcast**: Smotri.com broadcasts - - **smotri:community**: Smotri.com community videos - - **smotri:user**: Smotri.com user videos - **Snotr** - **Sohu** - **SonyLIV** @@ -883,7 +895,6 @@ - **Tagesschau** - **tagesschau:player** - **Tass** - - **TastyTrade** - **TBS** - **TDSLifeway** - **Teachable** @@ -906,6 +917,7 @@ - **TeleQuebecEmission** - **TeleQuebecLive** - **TeleQuebecSquat** + - **TeleQuebecVideo** - **TeleTask** - **Telewebion** - **TennisTV** @@ -923,6 +935,7 @@ - **ThisAV** - **ThisOldHouse** - **TikTok** + - **TikTokUser** (Currently broken) - **tinypic**: tinypic.com videos - **TMZ** - **TMZArticle** @@ -955,12 +968,15 @@ - **TV2DKBornholmPlay** - **TV4**: tv4.se and tv4play.se - **TV5MondePlus**: TV5MONDE+ + - **tv5unis** + - **tv5unis:video** - **tv8.it** - **TVA** - **TVANouvelles** - **TVANouvellesArticle** - **TVC** - **TVCArticle** + - **TVer** - **tvigle**: Интернет-телевидение Tvigle.ru - **tvland.com** - **TVN24** @@ -1089,6 +1105,7 @@ - **vube**: Vube.com - **VuClip** - **VVVVID** + - **VVVVIDShow** - **VyboryMos** - **Vzaar** - **Wakanim** @@ -1111,6 +1128,7 @@ - **WeiboMobile** - **WeiqiTV**: WQTV - **Wistia** + - **WistiaPlaylist** - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **WorldStarHipHop** - **WSJ**: Wall Street Journal @@ -1142,6 +1160,8 @@ - **yahoo:japannews**: Yahoo! Japan News - **YandexDisk** - **yandexmusic:album**: Яндекс.Музыка - Альбом + - **yandexmusic:artist:albums**: Яндекс.Музыка - Артист - Альбомы + - **yandexmusic:artist:tracks**: Яндекс.Музыка - Артист - Треки - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист - **yandexmusic:track**: Яндекс.Музыка - Трек - **YandexVideo** @@ -1169,9 +1189,9 @@ - **youtube:subscriptions**: YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication) - **youtube:tab**: YouTube.com tab - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication) + - **YoutubeYtBe**: youtu.be - **YoutubeYtUser**: YouTube.com user videos, URL or "ytuser" keyword - **Zapiks** - - **Zaq1** - **Zattoo** - **ZattooLive** - **ZDF-3sat** diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index bdd01e41a..22e3d26a7 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -98,6 +98,55 @@ class TestInfoExtractor(unittest.TestCase): self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True) self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True) + def test_search_json_ld_realworld(self): + # https://github.com/ytdl-org/youtube-dl/issues/23306 + expect_dict( + self, + self.ie._search_json_ld(r'''''', None), + { + 'title': '1 On 1 With Kleio', + 'description': 'Kleio Valentien', + 'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4', + 'timestamp': 1449347075, + 'duration': 743.0, + 'view_count': 1120958, + 'width': 1920, + 'height': 1080, + }) + def test_download_json(self): uri = encode_data_uri(b'{"foo": "blah"}', 'application/json') self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'}) @@ -108,6 +157,18 @@ class TestInfoExtractor(unittest.TestCase): self.assertEqual(self.ie._download_json(uri, None, fatal=False), None) def test_parse_html5_media_entries(self): + # inline video tag + expect_dict( + self, + self.ie._parse_html5_media_entries( + 'https://127.0.0.1/video.html', + r'