diff options
| -rw-r--r-- | CONTRIBUTORS | 16 | ||||
| -rw-r--r-- | Changelog.md | 211 | ||||
| -rw-r--r-- | devscripts/make_lazy_extractors.py | 11 | ||||
| -rw-r--r-- | supportedsites.md | 36 | ||||
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 4 | ||||
| -rw-r--r-- | yt_dlp/__init__.py | 1 | ||||
| -rw-r--r-- | yt_dlp/extractor/common.py | 13 | ||||
| -rw-r--r-- | yt_dlp/extractor/jamendo.py | 2 | ||||
| -rw-r--r-- | yt_dlp/extractor/trovo.py | 3 | ||||
| -rw-r--r-- | yt_dlp/extractor/youtube.py | 2 | ||||
| -rw-r--r-- | yt_dlp/options.py | 3 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/ffmpeg.py | 6 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/metadataparser.py | 13 | ||||
| -rw-r--r-- | yt_dlp/version.py | 4 |
14 files changed, 282 insertions, 43 deletions
diff --git a/CONTRIBUTORS b/CONTRIBUTORS index f035ce10d..b3e3e97fa 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -139,3 +139,19 @@ rhendric sdomi selfisekai stanoarn +0xA7404A/Aurora +4a1e2y5 +aarubui +chio0hai +cntrl-s +Deer-Spangle +DEvmIb +Grabien +j54vc1bk +mpeter50 +mrpapersonic +pabs3 +staubichsauger +xenova +Yakabuff +zulaport diff --git a/Changelog.md b/Changelog.md index c33691969..4e9a448cb 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,12 +5,115 @@ * Run `make doc` * Update Changelog.md and CONTRIBUTORS -* Change "Merged with ytdl" version in Readme.md if needed -* Add new/fixed extractors in "new features" section of Readme.md +* Change "Based on ytdl" version in Readme.md if needed * Commit as `Release <version>` and push to master * Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master --> +### 2021.12.01 + +* **Add option `--wait-for-video` to wait for scheduled streams** +* Add option `--break-per-input` to apply --break-on... to each input URL +* Add option `--embed-info-json` to embed info.json in mkv +* Add compat-option `embed-metadata` +* Allow using a custom format selector through API +* [AES] Add ECB mode by [nao20010128nao](https://github.com/nao20010128nao) +* [build] Fix MacOS Build +* [build] Save Git HEAD at release alongside version info +* [build] Use `workflow_dispatch` for release +* [downloader/ffmpeg] Fix for direct videos inside mpd manifests +* [downloader] Add colors to download progress +* [EmbedSubtitles] Slightly relax duration check and related cleanup +* [ExtractAudio] Fix conversion to `wav` and `vorbis` +* [ExtractAudio] Support `alac` +* [extractor] Extract `average_rating` from JSON-LD +* [FixupM3u8] Fixup MPEG-TS in MP4 container +* [generic] Support mpd manifests without extension by [shirt](https://github.com/shirt-dev) +* [hls] Better FairPlay DRM detection by [nyuszika7h](https://github.com/nyuszika7h) +* [jsinterp] Fix splice to handle float (for youtube js player f1ca6900) +* [utils] Allow alignment in `render_table` and add tests +* [utils] Fix `PagedList` +* [utils] Fix error when copying `LazyList` +* Clarify video/audio-only formats in -F +* Ensure directory exists when checking formats +* Ensure path for link files exists by [Zirro](https://github.com/Zirro) +* Ensure same config file is not loaded multiple times +* Fix `postprocessor_hooks` +* Fix `--break-on-archive` when pre-checking +* Fix `--check-formats` for `mhtml` +* Fix `--load-info-json` of playlists with failed entries +* Fix `--trim-filename` when filename has `.` +* Fix bug in parsing `--add-header` +* Fix error in `report_unplayable_conflict` by [shirt](https://github.com/shirt-dev) +* Fix writing playlist infojson with `--no-clean-infojson` +* Validate --get-bypass-country +* [blogger] Add extractor by [pabs3](https://github.com/pabs3) +* [breitbart] Add extractor by [Grabien](https://github.com/Grabien) +* [CableAV] Add extractor by [j54vc1bk](https://github.com/j54vc1bk) +* [CanalAlpha] Add extractor by [Ashish0804](https://github.com/Ashish0804) +* [CozyTV] Add extractor by [Ashish0804](https://github.com/Ashish0804) +* [CPTwentyFour] Add extractor by [Ashish0804](https://github.com/Ashish0804) +* [DiscoveryPlus] Add `DiscoveryPlusItalyShowIE` by [Ashish0804](https://github.com/Ashish0804) +* [ESPNCricInfo] Add extractor by [Ashish0804](https://github.com/Ashish0804) +* [LinkedIn] Add extractor by [u-spec-png](https://github.com/u-spec-png) +* [mixch] Add extractor by [nao20010128nao](https://github.com/nao20010128nao) +* [nebula] Add `NebulaCollectionIE` and rewrite extractor by [hheimbuerger](https://github.com/hheimbuerger) +* [OneFootball] Add extractor by [Ashish0804](https://github.com/Ashish0804) +* [peer.tv] Add extractor by [u-spec-png](https://github.com/u-spec-png) +* [radiozet] Add extractor by [0xA7404A](https://github.com/0xA7404A) (Aurora) +* [redgifs] Add extractor by [chio0hai](https://github.com/chio0hai) +* [RedGifs] Add Search and User extractors by [Deer-Spangle](https://github.com/Deer-Spangle) +* [rtrfm] Add extractor by [pabs3](https://github.com/pabs3) +* [Streamff] Add extractor by [cntrl-s](https://github.com/cntrl-s) +* [Stripchat] Add extractor by [zulaport](https://github.com/zulaport) +* [Aljazeera] Fix extractor by [u-spec-png](https://github.com/u-spec-png) +* [AmazonStoreIE] Fix regex to not match vdp urls by [Ashish0804](https://github.com/Ashish0804) +* [ARDBetaMediathek] Handle new URLs +* [bbc] Get all available formats by [nyuszika7h](https://github.com/nyuszika7h) +* [Bilibili] Fix title extraction by [u-spec-png](https://github.com/u-spec-png) +* [CBC Gem] Fix for shows that don't have all seasons by [makeworld-the-better-one](https://github.com/makeworld-the-better-one) +* [curiositystream] Add more metadata +* [CuriosityStream] Fix series +* [DiscoveryPlus] Rewrite extractors by [Ashish0804](https://github.com/Ashish0804), [pukkandan](https://github.com/pukkandan) +* [HotStar] Set language field from tags by [Ashish0804](https://github.com/Ashish0804) +* [instagram, cleanup] Refactor extractors +* [Instagram] Display more login errors by [MinePlayersPE](https://github.com/MinePlayersPE) +* [itv] Fix extractor by [staubichsauger](https://github.com/staubichsauger), [pukkandan](https://github.com/pukkandan) +* [mediaklikk] Expand valid URL +* [MTV] Improve mgid extraction by [Sipherdrakon](https://github.com/Sipherdrakon), [kikuyan](https://github.com/kikuyan) +* [nexx] Better error message for unsupported format +* [NovaEmbed] Fix extractor by [pukkandan](https://github.com/pukkandan), [std-move](https://github.com/std-move) +* [PatreonUser] Do not capture RSS URLs +* [Reddit] Add support for 1080p videos by [xenova](https://github.com/xenova) +* [RoosterTeethSeries] Fix for multiple pages by [MinePlayersPE](https://github.com/MinePlayersPE) +* [sbs] Fix for movies and livestreams +* [Senate.gov] Add SenateGovIE and fix SenateISVPIE by [Grabien](https://github.com/Grabien), [pukkandan](https://github.com/pukkandan) +* [soundcloud:search] Fix pagination +* [tiktok:user] Set `webpage_url` correctly +* [Tokentube] Fix description by [u-spec-png](https://github.com/u-spec-png) +* [trovo] Fix extractor by [nyuszika7h](https://github.com/nyuszika7h) +* [tv2] Expand valid URL +* [Tvplayhome] Fix extractor by [pukkandan](https://github.com/pukkandan), [18928172992817182](https://github.com/18928172992817182) +* [Twitch:vod] Add chapters by [mpeter50](https://github.com/mpeter50) +* [twitch:vod] Extract live status by [DEvmIb](https://github.com/DEvmIb) +* [VidLii] Add 720p support by [mrpapersonic](https://github.com/mrpapersonic) +* [vimeo] Add fallback for config URL +* [vimeo] Sort http formats higher +* [WDR] Expand valid URL +* [willow] Add extractor by [aarubui](https://github.com/aarubui) +* [xvideos] Detect embed URLs by [4a1e2y5](https://github.com/4a1e2y5) +* [xvideos] Fix extractor by [Yakabuff](https://github.com/Yakabuff) +* [youtube, cleanup] Reorganize Tab and Search extractor inheritances +* [youtube:search_url] Add playlist/channel support +* [youtube] Add `default` player client by [coletdjnz](https://github.com/coletdjnz) +* [youtube] Add storyboard formats +* [youtube] Decrypt n-sig for URLs with `ratebypass` +* [youtube] Minor improvement to format sorting +* [cleanup] Add deprecation warnings +* [cleanup] Refactor `JSInterpreter._seperate` +* [Cleanup] Remove some unnecessary groups in regexes by [Ashish0804](https://github.com/Ashish0804) +* [cleanup] Misc cleanup + ### 2021.11.10.1 @@ -1370,7 +1473,7 @@ * Cleaned up the fork for public use -**PS**: All uncredited changes above this point are authored by [pukkandan](https://github.com/pukkandan) +**Note**: All uncredited changes above this point are authored by [pukkandan](https://github.com/pukkandan) ### Unreleased changes in [blackjack4494/yt-dlc](https://github.com/blackjack4494/yt-dlc) * Updated to youtube-dl release 2020.11.26 by [pukkandan](https://github.com/pukkandan) @@ -1395,8 +1498,110 @@ * [spreaker] fix SpreakerShowIE test URL by [pukkandan](https://github.com/pukkandan) * [Vlive] Fix playlist handling when downloading a channel by [kyuyeunk](https://github.com/kyuyeunk) * [tmz] Fix extractor by [diegorodriguezv](https://github.com/diegorodriguezv) +* [ITV] BTCC URL update by [WolfganP](https://github.com/WolfganP) * [generic] Detect embedded bitchute videos by [pukkandan](https://github.com/pukkandan) * [generic] Extract embedded youtube and twitter videos by [diegorodriguezv](https://github.com/diegorodriguezv) * [ffmpeg] Ensure all streams are copied by [pukkandan](https://github.com/pukkandan) * [embedthumbnail] Fix for os.rename error by [pukkandan](https://github.com/pukkandan) * make_win.bat: don't use UPX to pack vcruntime140.dll by [jbruchon](https://github.com/jbruchon) + + +### Changelog of [blackjack4494/yt-dlc](https://github.com/blackjack4494/yt-dlc) till release 2020.11.11-3 + +**Note**: This was constructed from the merge commit messages and may not be entirely accurate + +* [bandcamp] fix failing test. remove subclass hack by [insaneracist](https://github.com/insaneracist) +* [bandcamp] restore album downloads by [insaneracist](https://github.com/insaneracist) +* [francetv] fix extractor by [Surkal](https://github.com/Surkal) +* [gdcvault] fix extractor by [blackjack4494](https://github.com/blackjack4494) +* [hotstar] Move to API v1 by [theincognito-inc](https://github.com/theincognito-inc) +* [hrfernsehen] add extractor by [blocktrron](https://github.com/blocktrron) +* [kakao] new apis by [blackjack4494](https://github.com/blackjack4494) +* [la7] fix missing protocol by [nixxo](https://github.com/nixxo) +* [mailru] removed escaped braces, use urljoin, added tests by [nixxo](https://github.com/nixxo) +* [MTV/Nick] universal mgid extractor + fix nick.de feed by [blackjack4494](https://github.com/blackjack4494) +* [mtv] Fix a missing match_id by [nixxo](https://github.com/nixxo) +* [Mtv] updated extractor logic & more by [blackjack4494](https://github.com/blackjack4494) +* [ndr] support Daserste ndr by [blackjack4494](https://github.com/blackjack4494) +* [Netzkino] Only use video id to find metadata by [TobiX](https://github.com/TobiX) +* [newgrounds] fix: video download by [insaneracist](https://github.com/insaneracist) +* [nitter] Add new extractor by [B0pol](https://github.com/B0pol) +* [soundcloud] Resolve audio/x-wav by [tfvlrue](https://github.com/tfvlrue) +* [soundcloud] sets pattern and tests by [blackjack4494](https://github.com/blackjack4494) +* [SouthparkDE/MTV] another mgid extraction (mtv_base) feed url updated by [blackjack4494](https://github.com/blackjack4494) +* [StoryFire] Add new extractor by [sgstair](https://github.com/sgstair) +* [twitch] by [geauxlo](https://github.com/geauxlo) +* [videa] Adapt to updates by [adrianheine](https://github.com/adrianheine) +* [Viki] subtitles, formats by [blackjack4494](https://github.com/blackjack4494) +* [vlive] fix extractor for revamped website by [exwm](https://github.com/exwm) +* [xtube] fix extractor by [insaneracist](https://github.com/insaneracist) +* [youtube] Convert subs when download is skipped by [blackjack4494](https://github.com/blackjack4494) +* [youtube] Fix age gate detection by [random-nick](https://github.com/random-nick) +* [youtube] fix yt-only playback when age restricted/gated - requires cookies by [blackjack4494](https://github.com/blackjack4494) +* [youtube] fix: extract artist metadata from ytInitialData by [insaneracist](https://github.com/insaneracist) +* [youtube] fix: extract mix playlist ids from ytInitialData by [insaneracist](https://github.com/insaneracist) +* [youtube] fix: mix playlist title by [insaneracist](https://github.com/insaneracist) +* [youtube] fix: Youtube Music playlists by [insaneracist](https://github.com/insaneracist) +* [Youtube] Fixed problem with new youtube player by [peet1993](https://github.com/peet1993) +* [zoom] Fix url parsing for url's containing /share/ and dots by [Romern](https://github.com/Romern) +* [zoom] new extractor by [insaneracist](https://github.com/insaneracist) +* abc by [adrianheine](https://github.com/adrianheine) +* Added Comcast_SSO fix by [merval](https://github.com/merval) +* Added DRM logic to brightcove by [merval](https://github.com/merval) +* Added regex for ABC.com site. by [kucksdorfs](https://github.com/kucksdorfs) +* alura by [hugohaa](https://github.com/hugohaa) +* Arbitrary merges by [fstirlitz](https://github.com/fstirlitz) +* ard.py_add_playlist_support by [martin54](https://github.com/martin54) +* Bugfix/youtube/chapters fix extractor by [gschizas](https://github.com/gschizas) +* bugfix_youtube_like_extraction by [RedpointsBots](https://github.com/RedpointsBots) +* Create build workflow by [blackjack4494](https://github.com/blackjack4494) +* deezer by [LucBerge](https://github.com/LucBerge) +* Detect embedded bitchute videos by [pukkandan](https://github.com/pukkandan) +* Don't install tests by [l29ah](https://github.com/l29ah) +* Don't try to embed/convert json subtitles generated by [youtube](https://github.com/youtube) livechat by [pukkandan](https://github.com/pukkandan) +* Doodstream by [sxvghd](https://github.com/sxvghd) +* duboku by [lkho](https://github.com/lkho) +* elonet by [tpikonen](https://github.com/tpikonen) +* ext/remuxe-video by [Zocker1999NET](https://github.com/Zocker1999NET) +* fall-back to the old way to fetch subtitles, if needed by [RobinD42](https://github.com/RobinD42) +* feature_subscriber_count by [RedpointsBots](https://github.com/RedpointsBots) +* Fix external downloader when there is no http_header by [pukkandan](https://github.com/pukkandan) +* Fix issue triggered by [tubeup](https://github.com/tubeup) by [nsapa](https://github.com/nsapa) +* Fix YoutubePlaylistsIE by [ZenulAbidin](https://github.com/ZenulAbidin) +* fix-mitele' by [DjMoren](https://github.com/DjMoren) +* fix/google-drive-cookie-issue by [legraphista](https://github.com/legraphista) +* fix_tiktok by [mervel-mervel](https://github.com/mervel-mervel) +* Fixed problem with JS player URL by [peet1993](https://github.com/peet1993) +* fixYTSearch by [xarantolus](https://github.com/xarantolus) +* FliegendeWurst-3sat-zdf-merger-bugfix-feature +* gilou-bandcamp_update +* implement ThisVid extractor by [rigstot](https://github.com/rigstot) +* JensTimmerman-patch-1 by [JensTimmerman](https://github.com/JensTimmerman) +* Keep download archive in memory for better performance by [jbruchon](https://github.com/jbruchon) +* la7-fix by [iamleot](https://github.com/iamleot) +* magenta by [adrianheine](https://github.com/adrianheine) +* Merge 26564 from [adrianheine](https://github.com/adrianheine) +* Merge code from [ddland](https://github.com/ddland) +* Merge code from [nixxo](https://github.com/nixxo) +* Merge code from [ssaqua](https://github.com/ssaqua) +* Merge code from [zubearc](https://github.com/zubearc) +* mkvthumbnail by [MrDoritos](https://github.com/MrDoritos) +* myvideo_ge by [fonkap](https://github.com/fonkap) +* naver by [SeonjaeHyeon](https://github.com/SeonjaeHyeon) +* ondemandkorea by [julien-hadleyjack](https://github.com/julien-hadleyjack) +* rai-update by [iamleot](https://github.com/iamleot) +* RFC: youtube: Polymer UI and JSON endpoints for playlists by [wlritchi](https://github.com/wlritchi) +* rutv by [adrianheine](https://github.com/adrianheine) +* Sc extractor web auth by [blackjack4494](https://github.com/blackjack4494) +* Switch from binary search tree to Python sets by [jbruchon](https://github.com/jbruchon) +* tiktok by [skyme5](https://github.com/skyme5) +* tvnow by [TinyToweringTree](https://github.com/TinyToweringTree) +* twitch-fix by [lel-amri](https://github.com/lel-amri) +* Twitter shortener by [blackjack4494](https://github.com/blackjack4494) +* Update README.md by [JensTimmerman](https://github.com/JensTimmerman) +* Update to reflect website changes. by [amigatomte](https://github.com/amigatomte) +* use webarchive to fix a dead link in README by [B0pol](https://github.com/B0pol) +* Viki the second by [blackjack4494](https://github.com/blackjack4494) +* wdr-subtitles by [mrtnmtth](https://github.com/mrtnmtth) +* Webpfix by [alexmerkel](https://github.com/alexmerkel) +* Youtube live chat by [siikamiika](https://github.com/siikamiika) diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 0411df76b..b58fb85e3 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -39,12 +39,6 @@ class {name}({bases}): _module = '{module}' ''' -make_valid_template = ''' - @classmethod - def _make_valid_url(cls): - return {valid_url!r} -''' - def get_base_name(base): if base is InfoExtractor: @@ -61,15 +55,14 @@ def build_lazy_ie(ie, name): bases=', '.join(map(get_base_name, ie.__bases__)), module=ie.__module__) valid_url = getattr(ie, '_VALID_URL', None) + if not valid_url and hasattr(ie, '_make_valid_url'): + valid_url = ie._make_valid_url() if valid_url: s += f' _VALID_URL = {valid_url!r}\n' if not ie._WORKING: s += ' _WORKING = False\n' if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: s += f'\n{getsource(ie.suitable)}' - if hasattr(ie, '_make_valid_url'): - # search extractors - s += make_valid_template.format(valid_url=ie._make_valid_url()) return s diff --git a/supportedsites.md b/supportedsites.md index 50fa7f9f1..2c13a28b7 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -141,6 +141,7 @@ - **BlackboardCollaborate** - **BleacherReport** - **BleacherReportCMS** + - **blogger.com** - **Bloomberg** - **BokeCC** - **BongaCams** @@ -150,6 +151,7 @@ - **BR**: Bayerischer Rundfunk - **BravoTV** - **Break** + - **BreitBart** - **brightcove:legacy** - **brightcove:new** - **BRMediathek**: Bayerischer Rundfunk Mediathek @@ -158,11 +160,13 @@ - **BusinessInsider** - **BuzzFeed** - **BYUtv** + - **CableAV** - **CAM4** - **Camdemy** - **CamdemyFolder** - **CamModels** - **CamWithHer** + - **CanalAlpha** - **canalc2.tv** - **Canalplus**: mycanal.fr and piwiplus.fr - **Canvas** @@ -222,6 +226,8 @@ - **CONtv** - **Corus** - **Coub** + - **CozyTV** + - **cp24** - **Cracked** - **Crackle** - **CrooksAndLiars** @@ -236,7 +242,8 @@ - **cu.ntv.co.jp**: Nippon Television Network - **CultureUnplugged** - **curiositystream** - - **curiositystream:collection** + - **curiositystream:collections** + - **curiositystream:series** - **CWTV** - **DagelijkseKost**: dagelijksekost.een.be - **DailyMail** @@ -266,6 +273,7 @@ - **DiscoveryPlus** - **DiscoveryPlusIndia** - **DiscoveryPlusIndiaShow** + - **DiscoveryPlusItalyShow** - **DiscoveryVR** - **Disney** - **DIYNetwork** @@ -315,6 +323,7 @@ - **Escapist** - **ESPN** - **ESPNArticle** + - **ESPNCricInfo** - **EsriVideo** - **Europa** - **EUScreen** @@ -448,7 +457,7 @@ - **IndavideoEmbed** - **InfoQ** - **Instagram** - - **instagram:tag**: Instagram hashtag search + - **instagram:tag**: Instagram hashtag search URLs - **instagram:user**: Instagram user profile - **InstagramIOS**: IOS instagram:// URL - **Internazionale** @@ -524,6 +533,7 @@ - **LineLive** - **LineLiveChannel** - **LineTV** + - **LinkedIn** - **linkedin:learning** - **linkedin:learning:course** - **LinuxAcademy** @@ -591,6 +601,7 @@ - **mirrativ** - **mirrativ:user** - **MiTele**: mitele.es + - **mixch** - **mixcloud** - **mixcloud:playlist** - **mixcloud:user** @@ -663,6 +674,7 @@ - **ndr:embed:base** - **NDTV** - **Nebula** + - **nebula:collection** - **NerdCubedFeed** - **netease:album**: 网易云音乐 - 专辑 - **netease:djradio**: 网易云音乐 - 电台 @@ -696,8 +708,8 @@ - **niconico**: ニコニコ動画 - **NiconicoPlaylist** - **NiconicoUser** - - **nicovideo:search**: Nico video searches; "nicosearch:" prefix - - **nicovideo:search:date**: Nico video searches, newest first; "nicosearchdate:" prefix + - **nicovideo:search**: Nico video search; "nicosearch:" prefix + - **nicovideo:search:date**: Nico video search, newest first; "nicosearchdate:" prefix - **nicovideo:search_url**: Nico video search URLs - **Nintendo** - **Nitter** @@ -746,6 +758,7 @@ - **OlympicsReplay** - **on24**: ON24 - **OnDemandKorea** + - **OneFootball** - **onet.pl** - **onet.tv** - **onet.tv:channel** @@ -788,6 +801,7 @@ - **PatreonUser** - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC) - **PearVideo** + - **peer.tv** - **PeerTube** - **PeerTube:Playlist** - **peloton** @@ -873,6 +887,7 @@ - **RadioJavan** - **radiokapital** - **radiokapital:show** + - **RadioZetPodcast** - **radlive** - **radlive:channel** - **radlive:season** @@ -897,7 +912,9 @@ - **RedBullTV** - **RedBullTVRrnContent** - **Reddit** - - **RedditR** + - **RedGifs** + - **RedGifsSearch**: Redgifs search + - **RedGifsUser**: Redgifs user - **RedTube** - **RegioTV** - **RENTV** @@ -921,6 +938,7 @@ - **rtl2:you** - **rtl2:you:series** - **RTP** + - **RTRFM** - **RTS**: RTS.ch - **rtve.es:alacarta**: RTVE a la carta - **rtve.es:infantil**: RTVE infantil @@ -960,6 +978,7 @@ - **SCTE** - **SCTECourse** - **Seeker** + - **SenateGov** - **SenateISVP** - **SendtoNews** - **Servus** @@ -1038,8 +1057,10 @@ - **Streamanity** - **streamcloud.eu** - **StreamCZ** + - **StreamFF** - **StreetVoice** - **StretchInternet** + - **Stripchat** - **stv:player** - **SunPorno** - **sverigesradio:episode** @@ -1311,6 +1332,7 @@ - **WeiboMobile** - **WeiqiTV**: WQTV - **whowatch** + - **Willow** - **WimTV** - **Wistia** - **WistiaPlaylist** @@ -1370,8 +1392,8 @@ - **youtube:history**: Youtube watch history; ":ythis" keyword (requires cookies) - **youtube:playlist**: YouTube playlists - **youtube:recommended**: YouTube recommended videos; ":ytrec" keyword - - **youtube:search**: YouTube searches; "ytsearch:" prefix - - **youtube:search:date**: YouTube searches, newest videos first; "ytsearchdate:" prefix + - **youtube:search**: YouTube search; "ytsearch:" prefix + - **youtube:search:date**: YouTube search, newest videos first; "ytsearchdate:" prefix - **youtube:search_url**: YouTube search URLs with sorting and filter support - **youtube:subscriptions**: YouTube subscriptions feed; ":ytsubs" keyword (requires cookies) - **youtube:tab**: YouTube Tabs diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 524994ab5..45500ab5a 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -96,6 +96,7 @@ from .utils import ( ReExtractInfo, register_socks_protocols, RejectedVideoReached, + remove_terminal_sequences, render_table, replace_extension, SameFileError, @@ -326,7 +327,7 @@ class YoutubeDL(object): bidi_workaround: Work around buggy terminals without bidirectional text support, using fridibi debug_printtraffic:Print out sent and received HTTP traffic - include_ads: Download ads as well + include_ads: Download ads as well (deprecated) default_search: Prepend this string if an input url is not valid. 'auto' for elaborate guessing encoding: Use this encoding instead of the system-specified. @@ -775,6 +776,7 @@ class YoutubeDL(object): def to_console_title(self, message): if not self.params.get('consoletitle', False): return + message = remove_terminal_sequences(message) if compat_os_name == 'nt': if ctypes.windll.kernel32.GetConsoleWindow(): # c_wchar_p() might not be necessary if `message` is diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 005136e20..bedb5f7ab 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -92,6 +92,7 @@ def _real_main(argv=None): if opts.batchfile is not None: try: if opts.batchfile == '-': + write_string('Reading URLs from stdin:\n') batchfd = sys.stdin else: batchfd = io.open( diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 37e69d409..597db63d1 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -466,6 +466,8 @@ class InfoExtractor(object): # we have cached the regexp for *this* class, whereas getattr would also # match the superclass if '_VALID_URL_RE' not in cls.__dict__: + if '_VALID_URL' not in cls.__dict__: + cls._VALID_URL = cls._make_valid_url() cls._VALID_URL_RE = re.compile(cls._VALID_URL) return cls._VALID_URL_RE.match(url) @@ -3658,17 +3660,8 @@ class SearchInfoExtractor(InfoExtractor): def _make_valid_url(cls): return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY - @classmethod - def suitable(cls, url): - return re.match(cls._make_valid_url(), url) is not None - def _real_extract(self, query): - mobj = re.match(self._make_valid_url(), query) - if mobj is None: - raise ExtractorError('Invalid search query "%s"' % query) - - prefix = mobj.group('prefix') - query = mobj.group('query') + prefix, query = self._match_valid_url(query).group('prefix', 'query') if prefix == '': return self._get_n_results(query, 1) elif prefix == 'all': diff --git a/yt_dlp/extractor/jamendo.py b/yt_dlp/extractor/jamendo.py index 1db7c64af..755d9703b 100644 --- a/yt_dlp/extractor/jamendo.py +++ b/yt_dlp/extractor/jamendo.py @@ -59,7 +59,7 @@ class JamendoIE(InfoExtractor): })[0] def _real_extract(self, url): - track_id, display_id = self._VALID_URL_RE.match(url).groups() + track_id, display_id = self._match_valid_url(url).groups() # webpage = self._download_webpage( # 'https://www.jamendo.com/track/' + track_id, track_id) # models = self._parse_json(self._html_search_regex( diff --git a/yt_dlp/extractor/trovo.py b/yt_dlp/extractor/trovo.py index 9d49840a5..127a5d2dc 100644 --- a/yt_dlp/extractor/trovo.py +++ b/yt_dlp/extractor/trovo.py @@ -108,6 +108,7 @@ class TrovoVodIE(TrovoBaseIE): 'comments': 'mincount:8', 'categories': ['Grand Theft Auto V'], }, + 'skip': '404' }, { 'url': 'https://trovo.live/clip/lc-5285890810184026005', 'only_matching': True, @@ -198,7 +199,7 @@ class TrovoVodIE(TrovoBaseIE): return info -class TrovoChannelBaseIE(InfoExtractor): +class TrovoChannelBaseIE(TrovoBaseIE): def _get_vod_json(self, page, uid): raise NotImplementedError('This method must be implemented by subclasses') diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index e4854bead..566edb38f 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -4545,7 +4545,7 @@ class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor): _TESTS = [] -class YoutubeSearchDateIE(SearchInfoExtractor, YoutubeTabBaseInfoExtractor): +class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor): IE_NAME = YoutubeSearchIE.IE_NAME + ':date' _SEARCH_KEY = 'ytsearchdate' IE_DESC = 'YouTube search, newest videos first' diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 2750a0f7b..0f807e805 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -1615,6 +1615,9 @@ def parseOpts(overrideArguments=None): current_path = os.path.join(path, '%s.conf' % package) config = _readOptions(current_path, default=None) if config is not None: + current_path = os.path.realpath(current_path) + if current_path in paths.values(): + return False configs[name], paths[name] = config, current_path return parser.parse_args(config)[0].ignoreconfig return False diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 609f97e47..73bbf7fb0 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -593,10 +593,16 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): return [], info filename = info['filepath'] + + # Disabled temporarily. There needs to be a way to overide this + # in case of duration actually mismatching in extractor + # See: https://github.com/yt-dlp/yt-dlp/issues/1870, https://github.com/yt-dlp/yt-dlp/issues/1385 + ''' if info.get('duration') and not info.get('__real_download') and self._duration_mismatch( self._get_real_video_duration(filename, False), info['duration']): self.to_screen(f'Skipping {self.pp_key()} since the real and expected durations mismatch') return [], info + ''' ext = info['ext'] sub_langs, sub_names, sub_filenames = [], [], [] diff --git a/yt_dlp/postprocessor/metadataparser.py b/yt_dlp/postprocessor/metadataparser.py index 54b2c5627..807cd305d 100644 --- a/yt_dlp/postprocessor/metadataparser.py +++ b/yt_dlp/postprocessor/metadataparser.py @@ -96,7 +96,6 @@ class MetadataParserPP(PostProcessor): return f -# Deprecated class MetadataFromFieldPP(MetadataParserPP): @classmethod def to_action(cls, f): @@ -106,19 +105,17 @@ class MetadataFromFieldPP(MetadataParserPP): return ( cls.Actions.INTERPRET, match.group('in').replace('\\:', ':'), - match.group('out')) + match.group('out'), + ) def __init__(self, downloader, formats): - super().__init__(self, downloader, [self.to_action(f) for f in formats]) - self.deprecation_warning( - 'yt_dlp.postprocessor.MetadataFromFieldPP is deprecated ' - 'and may be removed in a future version. Use yt_dlp.postprocessor.MetadataParserPP instead') + super().__init__(downloader, [self.to_action(f) for f in formats]) # Deprecated class MetadataFromTitlePP(MetadataParserPP): def __init__(self, downloader, titleformat): - super().__init__(self, downloader, [(self.Actions.INTERPRET, 'title', titleformat)]) + super().__init__(downloader, [(self.Actions.INTERPRET, 'title', titleformat)]) self.deprecation_warning( 'yt_dlp.postprocessor.MetadataFromTitlePP is deprecated ' - 'and may be removed in a future version. Use yt_dlp.postprocessor.MetadataParserPP instead') + 'and may be removed in a future version. Use yt_dlp.postprocessor.MetadataFromFieldPP instead') diff --git a/yt_dlp/version.py b/yt_dlp/version.py index aa8fd80a3..8c07d099e 100644 --- a/yt_dlp/version.py +++ b/yt_dlp/version.py @@ -1,5 +1,5 @@ # Autogenerated by devscripts/update-version.py -__version__ = '2021.11.10.1' +__version__ = '2021.12.01' -RELEASE_GIT_HEAD = '7144b697f' +RELEASE_GIT_HEAD = '91f071af6' |