diff options
-rw-r--r-- | .github/ISSUE_TEMPLATE/1_broken_site.md | 17 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE/2_site_support_request.md | 14 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE/3_site_feature_request.md | 13 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE/4_bug_report.md | 17 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE/5_feature_request.md | 14 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE/6_question.md | 6 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE_tmpl/1_broken_site.md | 12 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md | 8 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md | 6 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE_tmpl/4_bug_report.md | 12 | ||||
-rw-r--r-- | .github/ISSUE_TEMPLATE_tmpl/5_feature_request.md | 6 | ||||
-rw-r--r-- | .github/workflows/build.yml | 151 | ||||
-rw-r--r-- | .github/workflows/python-publish.yml.disable | 33 | ||||
-rw-r--r-- | .gitignore | 11 | ||||
-rw-r--r-- | .travis.yml | 12 | ||||
-rw-r--r-- | ChangeLog | 39 | ||||
-rw-r--r-- | MANIFEST.in | 6 | ||||
-rw-r--r-- | Makefile | 84 | ||||
-rw-r--r-- | README.md | 894 | ||||
-rwxr-xr-x | bin/youtube-dl | 6 | ||||
-rw-r--r-- | devscripts/bash-completion.in | 4 | ||||
-rwxr-xr-x | devscripts/bash-completion.py | 6 | ||||
-rw-r--r-- | devscripts/buildserver.py | 4 | ||||
-rw-r--r-- | devscripts/check-porn.py | 4 | ||||
-rw-r--r-- | devscripts/create-github-release.py | 6 | ||||
-rw-r--r-- | devscripts/fish-completion.in | 2 | ||||
-rwxr-xr-x | devscripts/fish-completion.py | 11 | ||||
-rw-r--r-- | devscripts/generate_aes_testdata.py | 4 | ||||
-rwxr-xr-x | devscripts/gh-pages/add-version.py | 6 | ||||
-rwxr-xr-x | devscripts/gh-pages/update-feed.py | 10 | ||||
-rwxr-xr-x | devscripts/gh-pages/update-sites.py | 6 | ||||
-rwxr-xr-x | devscripts/make_contributing.py | 20 | ||||
-rw-r--r-- | devscripts/make_issue_template.py | 6 | ||||
-rw-r--r-- | devscripts/make_lazy_extractors.py | 4 | ||||
-rwxr-xr-x | devscripts/make_readme.py | 4 | ||||
-rw-r--r-- | devscripts/make_supportedsites.py | 6 | ||||
-rw-r--r-- | devscripts/prepare_manpage.py | 6 | ||||
-rwxr-xr-x | devscripts/release.sh | 22 | ||||
-rw-r--r-- | devscripts/show-downloads-statistics.py | 10 | ||||
-rw-r--r-- | devscripts/zsh-completion.in | 8 | ||||
-rwxr-xr-x | devscripts/zsh-completion.py | 6 | ||||
-rw-r--r-- | docs/Makefile | 8 | ||||
-rw-r--r-- | docs/conf.py | 10 | ||||
-rw-r--r-- | docs/faq.md | 31 | ||||
-rw-r--r-- | docs/index.rst | 6 | ||||
-rw-r--r-- | docs/module_guide.rst | 8 | ||||
-rw-r--r-- | docs/supportedsites.md | 21 | ||||
-rw-r--r-- | make_win.bat | 1 | ||||
-rw-r--r-- | pyinst.py | 92 | ||||
-rw-r--r-- | pyinst32.py | 92 | ||||
-rw-r--r-- | scripts/update-version-workflow.py | 44 | ||||
-rw-r--r-- | scripts/update-version.py | 31 | ||||
-rw-r--r-- | setup.cfg | 2 | ||||
-rw-r--r-- | setup.py | 149 | ||||
-rw-r--r-- | test/helper.py | 10 | ||||
-rw-r--r-- | test/test_InfoExtractor.py | 8 | ||||
-rw-r--r-- | test/test_YoutubeDL.py | 12 | ||||
-rw-r--r-- | test/test_YoutubeDLCookieJar.py | 2 | ||||
-rw-r--r-- | test/test_aes.py | 4 | ||||
-rw-r--r-- | test/test_age_restriction.py | 2 | ||||
-rw-r--r-- | test/test_all_urls.py | 4 | ||||
-rw-r--r-- | test/test_cache.py | 2 | ||||
-rw-r--r-- | test/test_compat.py | 14 | ||||
-rw-r--r-- | test/test_download.py | 12 | ||||
-rw-r--r-- | test/test_downloader_http.py | 8 | ||||
-rw-r--r-- | test/test_execution.py | 10 | ||||
-rw-r--r-- | test/test_http.py | 4 | ||||
-rw-r--r-- | test/test_iqiyi_sdk_interpreter.py | 2 | ||||
-rw-r--r-- | test/test_jsinterp.py | 2 | ||||
-rw-r--r-- | test/test_netrc.py | 2 | ||||
-rw-r--r-- | test/test_options.py | 2 | ||||
-rw-r--r-- | test/test_postprocessors.py | 2 | ||||
-rw-r--r-- | test/test_socks.py | 2 | ||||
-rw-r--r-- | test/test_subtitles.py | 18 | ||||
-rw-r--r-- | test/test_swfinterp.py | 2 | ||||
-rw-r--r-- | test/test_unicode_literals.py | 2 | ||||
-rw-r--r-- | test/test_update.py | 2 | ||||
-rw-r--r-- | test/test_utils.py | 14 | ||||
-rw-r--r-- | test/test_verbose_output.py | 8 | ||||
-rw-r--r-- | test/test_write_annotations.py | 8 | ||||
-rw-r--r-- | test/test_youtube_chapters.py | 2 | ||||
-rw-r--r-- | test/test_youtube_lists.py | 2 | ||||
-rw-r--r-- | test/test_youtube_signature.py | 4 | ||||
-rw-r--r-- | tox.ini | 2 | ||||
-rw-r--r-- | win/icon/cloud.ico | bin | 0 -> 4286 bytes | |||
-rw-r--r-- | win/ver.txt | 45 | ||||
-rw-r--r-- | youtube-dl.plugin.zsh | 24 | ||||
-rw-r--r-- | youtube_dl/extractor/deezer.py | 91 | ||||
-rw-r--r-- | youtube_dl/extractor/dreisat.py | 193 | ||||
-rw-r--r-- | youtube_dl/extractor/phoenix.py | 45 | ||||
-rw-r--r-- | youtube_dl/extractor/redbulltv.py | 128 | ||||
-rw-r--r-- | youtube_dl/extractor/tiktok.py | 138 | ||||
-rw-r--r--[-rwxr-xr-x] | youtube_dlc/YoutubeDL.py (renamed from youtube_dl/YoutubeDL.py) | 232 | ||||
-rw-r--r-- | youtube_dlc/__init__.py (renamed from youtube_dl/__init__.py) | 17 | ||||
-rw-r--r--[-rwxr-xr-x] | youtube_dlc/__main__.py (renamed from youtube_dl/__main__.py) | 8 | ||||
-rw-r--r-- | youtube_dlc/aes.py (renamed from youtube_dl/aes.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/cache.py (renamed from youtube_dl/cache.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/compat.py (renamed from youtube_dl/compat.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/downloader/__init__.py (renamed from youtube_dl/downloader/__init__.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/downloader/common.py (renamed from youtube_dl/downloader/common.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/downloader/dash.py (renamed from youtube_dl/downloader/dash.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/downloader/external.py (renamed from youtube_dl/downloader/external.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/downloader/f4m.py (renamed from youtube_dl/downloader/f4m.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/downloader/fragment.py (renamed from youtube_dl/downloader/fragment.py) | 4 | ||||
-rw-r--r-- | youtube_dlc/downloader/hls.py (renamed from youtube_dl/downloader/hls.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/downloader/http.py (renamed from youtube_dl/downloader/http.py) | 22 | ||||
-rw-r--r-- | youtube_dlc/downloader/ism.py (renamed from youtube_dl/downloader/ism.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/downloader/rtmp.py (renamed from youtube_dl/downloader/rtmp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/downloader/rtsp.py (renamed from youtube_dl/downloader/rtsp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/downloader/youtube_live_chat.py | 94 | ||||
-rw-r--r-- | youtube_dlc/extractor/__init__.py (renamed from youtube_dl/extractor/__init__.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/abc.py (renamed from youtube_dl/extractor/abc.py) | 104 | ||||
-rw-r--r-- | youtube_dlc/extractor/abcnews.py (renamed from youtube_dl/extractor/abcnews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/abcotvs.py (renamed from youtube_dl/extractor/abcotvs.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/academicearth.py (renamed from youtube_dl/extractor/academicearth.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/acast.py (renamed from youtube_dl/extractor/acast.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/adn.py (renamed from youtube_dl/extractor/adn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/adobeconnect.py (renamed from youtube_dl/extractor/adobeconnect.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/adobepass.py (renamed from youtube_dl/extractor/adobepass.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/adobetv.py (renamed from youtube_dl/extractor/adobetv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/adultswim.py (renamed from youtube_dl/extractor/adultswim.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/aenetworks.py (renamed from youtube_dl/extractor/aenetworks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/afreecatv.py (renamed from youtube_dl/extractor/afreecatv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/airmozilla.py (renamed from youtube_dl/extractor/airmozilla.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/aliexpress.py (renamed from youtube_dl/extractor/aliexpress.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/aljazeera.py (renamed from youtube_dl/extractor/aljazeera.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/allocine.py (renamed from youtube_dl/extractor/allocine.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/alphaporno.py (renamed from youtube_dl/extractor/alphaporno.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/alura.py | 180 | ||||
-rw-r--r-- | youtube_dlc/extractor/amcnetworks.py (renamed from youtube_dl/extractor/amcnetworks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/americastestkitchen.py (renamed from youtube_dl/extractor/americastestkitchen.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/amp.py (renamed from youtube_dl/extractor/amp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/animeondemand.py (renamed from youtube_dl/extractor/animeondemand.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/anvato.py (renamed from youtube_dl/extractor/anvato.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/aol.py (renamed from youtube_dl/extractor/aol.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/apa.py (renamed from youtube_dl/extractor/apa.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/aparat.py (renamed from youtube_dl/extractor/aparat.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/appleconnect.py (renamed from youtube_dl/extractor/appleconnect.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/appletrailers.py (renamed from youtube_dl/extractor/appletrailers.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/archiveorg.py (renamed from youtube_dl/extractor/archiveorg.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ard.py (renamed from youtube_dl/extractor/ard.py) | 154 | ||||
-rw-r--r-- | youtube_dlc/extractor/arkena.py (renamed from youtube_dl/extractor/arkena.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/arte.py (renamed from youtube_dl/extractor/arte.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/asiancrush.py (renamed from youtube_dl/extractor/asiancrush.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/atresplayer.py (renamed from youtube_dl/extractor/atresplayer.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/atttechchannel.py (renamed from youtube_dl/extractor/atttechchannel.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/atvat.py (renamed from youtube_dl/extractor/atvat.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/audimedia.py (renamed from youtube_dl/extractor/audimedia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/audioboom.py (renamed from youtube_dl/extractor/audioboom.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/audiomack.py (renamed from youtube_dl/extractor/audiomack.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/awaan.py (renamed from youtube_dl/extractor/awaan.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/aws.py (renamed from youtube_dl/extractor/aws.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/azmedien.py (renamed from youtube_dl/extractor/azmedien.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/baidu.py (renamed from youtube_dl/extractor/baidu.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bandcamp.py (renamed from youtube_dl/extractor/bandcamp.py) | 38 | ||||
-rw-r--r-- | youtube_dlc/extractor/bbc.py (renamed from youtube_dl/extractor/bbc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/beampro.py (renamed from youtube_dl/extractor/beampro.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/beatport.py (renamed from youtube_dl/extractor/beatport.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/beeg.py (renamed from youtube_dl/extractor/beeg.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/behindkink.py (renamed from youtube_dl/extractor/behindkink.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bellmedia.py (renamed from youtube_dl/extractor/bellmedia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bet.py (renamed from youtube_dl/extractor/bet.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/bfi.py (renamed from youtube_dl/extractor/bfi.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bigflix.py (renamed from youtube_dl/extractor/bigflix.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bild.py (renamed from youtube_dl/extractor/bild.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bilibili.py (renamed from youtube_dl/extractor/bilibili.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/biobiochiletv.py (renamed from youtube_dl/extractor/biobiochiletv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/biqle.py (renamed from youtube_dl/extractor/biqle.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bitchute.py (renamed from youtube_dl/extractor/bitchute.py) | 12 | ||||
-rw-r--r-- | youtube_dlc/extractor/bleacherreport.py (renamed from youtube_dl/extractor/bleacherreport.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/blinkx.py (renamed from youtube_dl/extractor/blinkx.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bloomberg.py (renamed from youtube_dl/extractor/bloomberg.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bokecc.py (renamed from youtube_dl/extractor/bokecc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bostonglobe.py (renamed from youtube_dl/extractor/bostonglobe.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bpb.py (renamed from youtube_dl/extractor/bpb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/br.py (renamed from youtube_dl/extractor/br.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/bravotv.py (renamed from youtube_dl/extractor/bravotv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/breakcom.py (renamed from youtube_dl/extractor/breakcom.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/brightcove.py (renamed from youtube_dl/extractor/brightcove.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/businessinsider.py (renamed from youtube_dl/extractor/businessinsider.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/buzzfeed.py (renamed from youtube_dl/extractor/buzzfeed.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/byutv.py (renamed from youtube_dl/extractor/byutv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/c56.py (renamed from youtube_dl/extractor/c56.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/camdemy.py (renamed from youtube_dl/extractor/camdemy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cammodels.py (renamed from youtube_dl/extractor/cammodels.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/camtube.py (renamed from youtube_dl/extractor/camtube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/camwithher.py (renamed from youtube_dl/extractor/camwithher.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/canalc2.py (renamed from youtube_dl/extractor/canalc2.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/canalplus.py (renamed from youtube_dl/extractor/canalplus.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/canvas.py (renamed from youtube_dl/extractor/canvas.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/carambatv.py (renamed from youtube_dl/extractor/carambatv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cartoonnetwork.py (renamed from youtube_dl/extractor/cartoonnetwork.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cbc.py (renamed from youtube_dl/extractor/cbc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cbs.py (renamed from youtube_dl/extractor/cbs.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cbsinteractive.py (renamed from youtube_dl/extractor/cbsinteractive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cbslocal.py (renamed from youtube_dl/extractor/cbslocal.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cbsnews.py (renamed from youtube_dl/extractor/cbsnews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cbssports.py (renamed from youtube_dl/extractor/cbssports.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ccc.py (renamed from youtube_dl/extractor/ccc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ccma.py (renamed from youtube_dl/extractor/ccma.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cctv.py (renamed from youtube_dl/extractor/cctv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cda.py (renamed from youtube_dl/extractor/cda.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ceskatelevize.py (renamed from youtube_dl/extractor/ceskatelevize.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/channel9.py (renamed from youtube_dl/extractor/channel9.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/charlierose.py (renamed from youtube_dl/extractor/charlierose.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/chaturbate.py (renamed from youtube_dl/extractor/chaturbate.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/chilloutzone.py (renamed from youtube_dl/extractor/chilloutzone.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/chirbit.py (renamed from youtube_dl/extractor/chirbit.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cinchcast.py (renamed from youtube_dl/extractor/cinchcast.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cinemax.py (renamed from youtube_dl/extractor/cinemax.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ciscolive.py (renamed from youtube_dl/extractor/ciscolive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cjsw.py (renamed from youtube_dl/extractor/cjsw.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cliphunter.py (renamed from youtube_dl/extractor/cliphunter.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/clippit.py (renamed from youtube_dl/extractor/clippit.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cliprs.py (renamed from youtube_dl/extractor/cliprs.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/clipsyndicate.py (renamed from youtube_dl/extractor/clipsyndicate.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/closertotruth.py (renamed from youtube_dl/extractor/closertotruth.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cloudflarestream.py (renamed from youtube_dl/extractor/cloudflarestream.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cloudy.py (renamed from youtube_dl/extractor/cloudy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/clubic.py (renamed from youtube_dl/extractor/clubic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/clyp.py (renamed from youtube_dl/extractor/clyp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cmt.py (renamed from youtube_dl/extractor/cmt.py) | 6 | ||||
-rw-r--r-- | youtube_dlc/extractor/cnbc.py (renamed from youtube_dl/extractor/cnbc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cnn.py (renamed from youtube_dl/extractor/cnn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/comedycentral.py (renamed from youtube_dl/extractor/comedycentral.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/common.py (renamed from youtube_dl/extractor/common.py) | 22 | ||||
-rw-r--r-- | youtube_dlc/extractor/commonmistakes.py (renamed from youtube_dl/extractor/commonmistakes.py) | 4 | ||||
-rw-r--r-- | youtube_dlc/extractor/commonprotocols.py (renamed from youtube_dl/extractor/commonprotocols.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/condenast.py (renamed from youtube_dl/extractor/condenast.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/contv.py (renamed from youtube_dl/extractor/contv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/corus.py (renamed from youtube_dl/extractor/corus.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/coub.py (renamed from youtube_dl/extractor/coub.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cracked.py (renamed from youtube_dl/extractor/cracked.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/crackle.py (renamed from youtube_dl/extractor/crackle.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/crooksandliars.py (renamed from youtube_dl/extractor/crooksandliars.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/crunchyroll.py (renamed from youtube_dl/extractor/crunchyroll.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cspan.py (renamed from youtube_dl/extractor/cspan.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ctsnews.py (renamed from youtube_dl/extractor/ctsnews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ctvnews.py (renamed from youtube_dl/extractor/ctvnews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cultureunplugged.py (renamed from youtube_dl/extractor/cultureunplugged.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/curiositystream.py (renamed from youtube_dl/extractor/curiositystream.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/cwtv.py (renamed from youtube_dl/extractor/cwtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dailymail.py (renamed from youtube_dl/extractor/dailymail.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dailymotion.py (renamed from youtube_dl/extractor/dailymotion.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/daum.py (renamed from youtube_dl/extractor/daum.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dbtv.py (renamed from youtube_dl/extractor/dbtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dctp.py (renamed from youtube_dl/extractor/dctp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/deezer.py | 147 | ||||
-rw-r--r-- | youtube_dlc/extractor/defense.py (renamed from youtube_dl/extractor/defense.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/democracynow.py (renamed from youtube_dl/extractor/democracynow.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dfb.py (renamed from youtube_dl/extractor/dfb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dhm.py (renamed from youtube_dl/extractor/dhm.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/digg.py (renamed from youtube_dl/extractor/digg.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/digiteka.py (renamed from youtube_dl/extractor/digiteka.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/discovery.py (renamed from youtube_dl/extractor/discovery.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/discoverygo.py (renamed from youtube_dl/extractor/discoverygo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/discoverynetworks.py (renamed from youtube_dl/extractor/discoverynetworks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/discoveryvr.py (renamed from youtube_dl/extractor/discoveryvr.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/disney.py (renamed from youtube_dl/extractor/disney.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dispeak.py (renamed from youtube_dl/extractor/dispeak.py) | 1 | ||||
-rw-r--r-- | youtube_dlc/extractor/dlive.py (renamed from youtube_dl/extractor/dlive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/doodstream.py | 71 | ||||
-rw-r--r-- | youtube_dlc/extractor/dotsub.py (renamed from youtube_dl/extractor/dotsub.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/douyutv.py (renamed from youtube_dl/extractor/douyutv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dplay.py (renamed from youtube_dl/extractor/dplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/drbonanza.py (renamed from youtube_dl/extractor/drbonanza.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dropbox.py (renamed from youtube_dl/extractor/dropbox.py) | 4 | ||||
-rw-r--r-- | youtube_dlc/extractor/drtuber.py (renamed from youtube_dl/extractor/drtuber.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/drtv.py (renamed from youtube_dl/extractor/drtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dtube.py (renamed from youtube_dl/extractor/dtube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/duboku.py | 242 | ||||
-rw-r--r-- | youtube_dlc/extractor/dumpert.py (renamed from youtube_dl/extractor/dumpert.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dvtv.py (renamed from youtube_dl/extractor/dvtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/dw.py (renamed from youtube_dl/extractor/dw.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/eagleplatform.py (renamed from youtube_dl/extractor/eagleplatform.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ebaumsworld.py (renamed from youtube_dl/extractor/ebaumsworld.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/echomsk.py (renamed from youtube_dl/extractor/echomsk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/egghead.py (renamed from youtube_dl/extractor/egghead.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ehow.py (renamed from youtube_dl/extractor/ehow.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/eighttracks.py (renamed from youtube_dl/extractor/eighttracks.py) | 22 | ||||
-rw-r--r-- | youtube_dlc/extractor/einthusan.py (renamed from youtube_dl/extractor/einthusan.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/eitb.py (renamed from youtube_dl/extractor/eitb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ellentube.py (renamed from youtube_dl/extractor/ellentube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/elonet.py | 137 | ||||
-rw-r--r-- | youtube_dlc/extractor/elpais.py (renamed from youtube_dl/extractor/elpais.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/embedly.py (renamed from youtube_dl/extractor/embedly.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/engadget.py (renamed from youtube_dl/extractor/engadget.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/eporner.py (renamed from youtube_dl/extractor/eporner.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/eroprofile.py (renamed from youtube_dl/extractor/eroprofile.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/escapist.py (renamed from youtube_dl/extractor/escapist.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/espn.py (renamed from youtube_dl/extractor/espn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/esri.py (renamed from youtube_dl/extractor/esri.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/europa.py (renamed from youtube_dl/extractor/europa.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/everyonesmixtape.py (renamed from youtube_dl/extractor/everyonesmixtape.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/expotv.py (renamed from youtube_dl/extractor/expotv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/expressen.py (renamed from youtube_dl/extractor/expressen.py) | 7 | ||||
-rw-r--r-- | youtube_dlc/extractor/extractors.py (renamed from youtube_dl/extractor/extractors.py) | 42 | ||||
-rw-r--r-- | youtube_dlc/extractor/extremetube.py (renamed from youtube_dl/extractor/extremetube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/eyedotv.py (renamed from youtube_dl/extractor/eyedotv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/facebook.py (renamed from youtube_dl/extractor/facebook.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/faz.py (renamed from youtube_dl/extractor/faz.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fc2.py (renamed from youtube_dl/extractor/fc2.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fczenit.py (renamed from youtube_dl/extractor/fczenit.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/filmon.py (renamed from youtube_dl/extractor/filmon.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/filmweb.py (renamed from youtube_dl/extractor/filmweb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/firsttv.py (renamed from youtube_dl/extractor/firsttv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fivemin.py (renamed from youtube_dl/extractor/fivemin.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fivetv.py (renamed from youtube_dl/extractor/fivetv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/flickr.py (renamed from youtube_dl/extractor/flickr.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/folketinget.py (renamed from youtube_dl/extractor/folketinget.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/footyroom.py (renamed from youtube_dl/extractor/footyroom.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/formula1.py (renamed from youtube_dl/extractor/formula1.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fourtube.py (renamed from youtube_dl/extractor/fourtube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fox.py (renamed from youtube_dl/extractor/fox.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fox9.py (renamed from youtube_dl/extractor/fox9.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/foxgay.py (renamed from youtube_dl/extractor/foxgay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/foxnews.py (renamed from youtube_dl/extractor/foxnews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/foxsports.py (renamed from youtube_dl/extractor/foxsports.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/franceculture.py (renamed from youtube_dl/extractor/franceculture.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/franceinter.py (renamed from youtube_dl/extractor/franceinter.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/francetv.py (renamed from youtube_dl/extractor/francetv.py) | 12 | ||||
-rw-r--r-- | youtube_dlc/extractor/freesound.py (renamed from youtube_dl/extractor/freesound.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/freespeech.py (renamed from youtube_dl/extractor/freespeech.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/freshlive.py (renamed from youtube_dl/extractor/freshlive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/frontendmasters.py (renamed from youtube_dl/extractor/frontendmasters.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/funimation.py (renamed from youtube_dl/extractor/funimation.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/funk.py (renamed from youtube_dl/extractor/funk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fusion.py (renamed from youtube_dl/extractor/fusion.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/fxnetworks.py (renamed from youtube_dl/extractor/fxnetworks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gaia.py (renamed from youtube_dl/extractor/gaia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gameinformer.py (renamed from youtube_dl/extractor/gameinformer.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gamespot.py (renamed from youtube_dl/extractor/gamespot.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gamestar.py (renamed from youtube_dl/extractor/gamestar.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gaskrank.py (renamed from youtube_dl/extractor/gaskrank.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gazeta.py (renamed from youtube_dl/extractor/gazeta.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gdcvault.py (renamed from youtube_dl/extractor/gdcvault.py) | 73 | ||||
-rw-r--r-- | youtube_dlc/extractor/generic.py (renamed from youtube_dl/extractor/generic.py) | 10 | ||||
-rw-r--r-- | youtube_dlc/extractor/gfycat.py (renamed from youtube_dl/extractor/gfycat.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/giantbomb.py (renamed from youtube_dl/extractor/giantbomb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/giga.py (renamed from youtube_dl/extractor/giga.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gigya.py (renamed from youtube_dl/extractor/gigya.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/glide.py (renamed from youtube_dl/extractor/glide.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/globo.py (renamed from youtube_dl/extractor/globo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/go.py (renamed from youtube_dl/extractor/go.py) | 6 | ||||
-rw-r--r-- | youtube_dlc/extractor/godtube.py (renamed from youtube_dl/extractor/godtube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/golem.py (renamed from youtube_dl/extractor/golem.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/googledrive.py (renamed from youtube_dl/extractor/googledrive.py) | 29 | ||||
-rw-r--r-- | youtube_dlc/extractor/googleplus.py (renamed from youtube_dl/extractor/googleplus.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/googlesearch.py (renamed from youtube_dl/extractor/googlesearch.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/goshgay.py (renamed from youtube_dl/extractor/goshgay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/gputechconf.py (renamed from youtube_dl/extractor/gputechconf.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/groupon.py (renamed from youtube_dl/extractor/groupon.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hbo.py (renamed from youtube_dl/extractor/hbo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hearthisat.py (renamed from youtube_dl/extractor/hearthisat.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/heise.py (renamed from youtube_dl/extractor/heise.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hellporno.py (renamed from youtube_dl/extractor/hellporno.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/helsinki.py (renamed from youtube_dl/extractor/helsinki.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hentaistigma.py (renamed from youtube_dl/extractor/hentaistigma.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hgtv.py (renamed from youtube_dl/extractor/hgtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hidive.py (renamed from youtube_dl/extractor/hidive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/historicfilms.py (renamed from youtube_dl/extractor/historicfilms.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hitbox.py (renamed from youtube_dl/extractor/hitbox.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hitrecord.py (renamed from youtube_dl/extractor/hitrecord.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hketv.py (renamed from youtube_dl/extractor/hketv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hornbunny.py (renamed from youtube_dl/extractor/hornbunny.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hotnewhiphop.py (renamed from youtube_dl/extractor/hotnewhiphop.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hotstar.py (renamed from youtube_dl/extractor/hotstar.py) | 57 | ||||
-rw-r--r-- | youtube_dlc/extractor/howcast.py (renamed from youtube_dl/extractor/howcast.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/howstuffworks.py (renamed from youtube_dl/extractor/howstuffworks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hrfensehen.py | 102 | ||||
-rw-r--r-- | youtube_dlc/extractor/hrti.py (renamed from youtube_dl/extractor/hrti.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/huajiao.py (renamed from youtube_dl/extractor/huajiao.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/huffpost.py (renamed from youtube_dl/extractor/huffpost.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hungama.py (renamed from youtube_dl/extractor/hungama.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/hypem.py (renamed from youtube_dl/extractor/hypem.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ign.py (renamed from youtube_dl/extractor/ign.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/imdb.py (renamed from youtube_dl/extractor/imdb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/imggaming.py (renamed from youtube_dl/extractor/imggaming.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/imgur.py (renamed from youtube_dl/extractor/imgur.py) | 4 | ||||
-rw-r--r-- | youtube_dlc/extractor/ina.py (renamed from youtube_dl/extractor/ina.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/inc.py (renamed from youtube_dl/extractor/inc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/indavideo.py (renamed from youtube_dl/extractor/indavideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/infoq.py (renamed from youtube_dl/extractor/infoq.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/instagram.py (renamed from youtube_dl/extractor/instagram.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/internazionale.py (renamed from youtube_dl/extractor/internazionale.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/internetvideoarchive.py (renamed from youtube_dl/extractor/internetvideoarchive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/iprima.py (renamed from youtube_dl/extractor/iprima.py) | 3 | ||||
-rw-r--r-- | youtube_dlc/extractor/iqiyi.py (renamed from youtube_dl/extractor/iqiyi.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ir90tv.py (renamed from youtube_dl/extractor/ir90tv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/itv.py (renamed from youtube_dl/extractor/itv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ivi.py (renamed from youtube_dl/extractor/ivi.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/ivideon.py (renamed from youtube_dl/extractor/ivideon.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/iwara.py (renamed from youtube_dl/extractor/iwara.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/izlesene.py (renamed from youtube_dl/extractor/izlesene.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/jamendo.py (renamed from youtube_dl/extractor/jamendo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/jeuxvideo.py (renamed from youtube_dl/extractor/jeuxvideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/joj.py (renamed from youtube_dl/extractor/joj.py) | 216 | ||||
-rw-r--r-- | youtube_dlc/extractor/jove.py (renamed from youtube_dl/extractor/jove.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/jwplatform.py (renamed from youtube_dl/extractor/jwplatform.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kakao.py (renamed from youtube_dl/extractor/kakao.py) | 35 | ||||
-rw-r--r-- | youtube_dlc/extractor/kaltura.py (renamed from youtube_dl/extractor/kaltura.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kanalplay.py (renamed from youtube_dl/extractor/kanalplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kankan.py (renamed from youtube_dl/extractor/kankan.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/karaoketv.py (renamed from youtube_dl/extractor/karaoketv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/karrierevideos.py (renamed from youtube_dl/extractor/karrierevideos.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/keezmovies.py (renamed from youtube_dl/extractor/keezmovies.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ketnet.py (renamed from youtube_dl/extractor/ketnet.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/khanacademy.py (renamed from youtube_dl/extractor/khanacademy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kickstarter.py (renamed from youtube_dl/extractor/kickstarter.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kinja.py (renamed from youtube_dl/extractor/kinja.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kinopoisk.py (renamed from youtube_dl/extractor/kinopoisk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/konserthusetplay.py (renamed from youtube_dl/extractor/konserthusetplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/krasview.py (renamed from youtube_dl/extractor/krasview.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ku6.py (renamed from youtube_dl/extractor/ku6.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kusi.py (renamed from youtube_dl/extractor/kusi.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/kuwo.py (renamed from youtube_dl/extractor/kuwo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/la7.py (renamed from youtube_dl/extractor/la7.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/laola1tv.py (renamed from youtube_dl/extractor/laola1tv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lci.py (renamed from youtube_dl/extractor/lci.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lcp.py (renamed from youtube_dl/extractor/lcp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lecture2go.py (renamed from youtube_dl/extractor/lecture2go.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lecturio.py (renamed from youtube_dl/extractor/lecturio.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/leeco.py (renamed from youtube_dl/extractor/leeco.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lego.py (renamed from youtube_dl/extractor/lego.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lemonde.py (renamed from youtube_dl/extractor/lemonde.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lenta.py (renamed from youtube_dl/extractor/lenta.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/libraryofcongress.py (renamed from youtube_dl/extractor/libraryofcongress.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/libsyn.py (renamed from youtube_dl/extractor/libsyn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lifenews.py (renamed from youtube_dl/extractor/lifenews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/limelight.py (renamed from youtube_dl/extractor/limelight.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/line.py (renamed from youtube_dl/extractor/line.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/linkedin.py (renamed from youtube_dl/extractor/linkedin.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/linuxacademy.py (renamed from youtube_dl/extractor/linuxacademy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/litv.py (renamed from youtube_dl/extractor/litv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/livejournal.py (renamed from youtube_dl/extractor/livejournal.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/liveleak.py (renamed from youtube_dl/extractor/liveleak.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/livestream.py (renamed from youtube_dl/extractor/livestream.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lnkgo.py (renamed from youtube_dl/extractor/lnkgo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/localnews8.py (renamed from youtube_dl/extractor/localnews8.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lovehomeporn.py (renamed from youtube_dl/extractor/lovehomeporn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lrt.py (renamed from youtube_dl/extractor/lrt.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/lynda.py (renamed from youtube_dl/extractor/lynda.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/m6.py (renamed from youtube_dl/extractor/m6.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/magentamusik360.py | 61 | ||||
-rw-r--r-- | youtube_dlc/extractor/mailru.py (renamed from youtube_dl/extractor/mailru.py) | 14 | ||||
-rw-r--r-- | youtube_dlc/extractor/malltv.py (renamed from youtube_dl/extractor/malltv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mangomolo.py (renamed from youtube_dl/extractor/mangomolo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/manyvids.py (renamed from youtube_dl/extractor/manyvids.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/markiza.py (renamed from youtube_dl/extractor/markiza.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/massengeschmacktv.py (renamed from youtube_dl/extractor/massengeschmacktv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/matchtv.py (renamed from youtube_dl/extractor/matchtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mdr.py (renamed from youtube_dl/extractor/mdr.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/medialaan.py (renamed from youtube_dl/extractor/medialaan.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mediaset.py (renamed from youtube_dl/extractor/mediaset.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mediasite.py (renamed from youtube_dl/extractor/mediasite.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/medici.py (renamed from youtube_dl/extractor/medici.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/megaphone.py (renamed from youtube_dl/extractor/megaphone.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/meipai.py (renamed from youtube_dl/extractor/meipai.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/melonvod.py (renamed from youtube_dl/extractor/melonvod.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/meta.py (renamed from youtube_dl/extractor/meta.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/metacafe.py (renamed from youtube_dl/extractor/metacafe.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/metacritic.py (renamed from youtube_dl/extractor/metacritic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mgoon.py (renamed from youtube_dl/extractor/mgoon.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mgtv.py (renamed from youtube_dl/extractor/mgtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/miaopai.py (renamed from youtube_dl/extractor/miaopai.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/microsoftvirtualacademy.py (renamed from youtube_dl/extractor/microsoftvirtualacademy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ministrygrid.py (renamed from youtube_dl/extractor/ministrygrid.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/minoto.py (renamed from youtube_dl/extractor/minoto.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/miomio.py (renamed from youtube_dl/extractor/miomio.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mit.py (renamed from youtube_dl/extractor/mit.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mitele.py (renamed from youtube_dl/extractor/mitele.py) | 41 | ||||
-rw-r--r-- | youtube_dlc/extractor/mixcloud.py (renamed from youtube_dl/extractor/mixcloud.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mlb.py (renamed from youtube_dl/extractor/mlb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mnet.py (renamed from youtube_dl/extractor/mnet.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/moevideo.py (renamed from youtube_dl/extractor/moevideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mofosex.py (renamed from youtube_dl/extractor/mofosex.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mojvideo.py (renamed from youtube_dl/extractor/mojvideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/morningstar.py (renamed from youtube_dl/extractor/morningstar.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/motherless.py (renamed from youtube_dl/extractor/motherless.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/motorsport.py (renamed from youtube_dl/extractor/motorsport.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/movieclips.py (renamed from youtube_dl/extractor/movieclips.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/moviezine.py (renamed from youtube_dl/extractor/moviezine.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/movingimage.py (renamed from youtube_dl/extractor/movingimage.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/msn.py (renamed from youtube_dl/extractor/msn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mtv.py (renamed from youtube_dl/extractor/mtv.py) | 46 | ||||
-rw-r--r-- | youtube_dlc/extractor/muenchentv.py (renamed from youtube_dl/extractor/muenchentv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mwave.py (renamed from youtube_dl/extractor/mwave.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/mychannels.py (renamed from youtube_dl/extractor/mychannels.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/myspace.py (renamed from youtube_dl/extractor/myspace.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/myspass.py (renamed from youtube_dl/extractor/myspass.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/myvi.py (renamed from youtube_dl/extractor/myvi.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/myvideoge.py | 56 | ||||
-rw-r--r-- | youtube_dlc/extractor/myvidster.py (renamed from youtube_dl/extractor/myvidster.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nationalgeographic.py (renamed from youtube_dl/extractor/nationalgeographic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/naver.py (renamed from youtube_dl/extractor/naver.py) | 85 | ||||
-rw-r--r-- | youtube_dlc/extractor/nba.py (renamed from youtube_dl/extractor/nba.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nbc.py (renamed from youtube_dl/extractor/nbc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ndr.py (renamed from youtube_dl/extractor/ndr.py) | 15 | ||||
-rw-r--r-- | youtube_dlc/extractor/ndtv.py (renamed from youtube_dl/extractor/ndtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nerdcubed.py (renamed from youtube_dl/extractor/nerdcubed.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/neteasemusic.py (renamed from youtube_dl/extractor/neteasemusic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/netzkino.py (renamed from youtube_dl/extractor/netzkino.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/newgrounds.py (renamed from youtube_dl/extractor/newgrounds.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/newstube.py (renamed from youtube_dl/extractor/newstube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nextmedia.py (renamed from youtube_dl/extractor/nextmedia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nexx.py (renamed from youtube_dl/extractor/nexx.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nfl.py (renamed from youtube_dl/extractor/nfl.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nhk.py (renamed from youtube_dl/extractor/nhk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nhl.py (renamed from youtube_dl/extractor/nhl.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nick.py (renamed from youtube_dl/extractor/nick.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/niconico.py (renamed from youtube_dl/extractor/niconico.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ninecninemedia.py (renamed from youtube_dl/extractor/ninecninemedia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ninegag.py (renamed from youtube_dl/extractor/ninegag.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ninenow.py (renamed from youtube_dl/extractor/ninenow.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nintendo.py (renamed from youtube_dl/extractor/nintendo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/njpwworld.py (renamed from youtube_dl/extractor/njpwworld.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nobelprize.py (renamed from youtube_dl/extractor/nobelprize.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/noco.py (renamed from youtube_dl/extractor/noco.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nonktube.py (renamed from youtube_dl/extractor/nonktube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/noovo.py (renamed from youtube_dl/extractor/noovo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/normalboots.py (renamed from youtube_dl/extractor/normalboots.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nosvideo.py (renamed from youtube_dl/extractor/nosvideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nova.py (renamed from youtube_dl/extractor/nova.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nowness.py (renamed from youtube_dl/extractor/nowness.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/noz.py (renamed from youtube_dl/extractor/noz.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/npo.py (renamed from youtube_dl/extractor/npo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/npr.py (renamed from youtube_dl/extractor/npr.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nrk.py (renamed from youtube_dl/extractor/nrk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nrl.py (renamed from youtube_dl/extractor/nrl.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ntvcojp.py (renamed from youtube_dl/extractor/ntvcojp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ntvde.py (renamed from youtube_dl/extractor/ntvde.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ntvru.py (renamed from youtube_dl/extractor/ntvru.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nuevo.py (renamed from youtube_dl/extractor/nuevo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nuvid.py (renamed from youtube_dl/extractor/nuvid.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nytimes.py (renamed from youtube_dl/extractor/nytimes.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/nzz.py (renamed from youtube_dl/extractor/nzz.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/odatv.py (renamed from youtube_dl/extractor/odatv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/odnoklassniki.py (renamed from youtube_dl/extractor/odnoklassniki.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/oktoberfesttv.py (renamed from youtube_dl/extractor/oktoberfesttv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/once.py (renamed from youtube_dl/extractor/once.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ondemandkorea.py (renamed from youtube_dl/extractor/ondemandkorea.py) | 38 | ||||
-rw-r--r-- | youtube_dlc/extractor/onet.py (renamed from youtube_dl/extractor/onet.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/onionstudios.py (renamed from youtube_dl/extractor/onionstudios.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ooyala.py (renamed from youtube_dl/extractor/ooyala.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/openload.py (renamed from youtube_dl/extractor/openload.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ora.py (renamed from youtube_dl/extractor/ora.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/orf.py (renamed from youtube_dl/extractor/orf.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/outsidetv.py (renamed from youtube_dl/extractor/outsidetv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/packtpub.py (renamed from youtube_dl/extractor/packtpub.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pandoratv.py (renamed from youtube_dl/extractor/pandoratv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/parliamentliveuk.py (renamed from youtube_dl/extractor/parliamentliveuk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/patreon.py (renamed from youtube_dl/extractor/patreon.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pbs.py (renamed from youtube_dl/extractor/pbs.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pearvideo.py (renamed from youtube_dl/extractor/pearvideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/peertube.py (renamed from youtube_dl/extractor/peertube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/people.py (renamed from youtube_dl/extractor/people.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/performgroup.py (renamed from youtube_dl/extractor/performgroup.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/periscope.py (renamed from youtube_dl/extractor/periscope.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/philharmoniedeparis.py (renamed from youtube_dl/extractor/philharmoniedeparis.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/phoenix.py | 52 | ||||
-rw-r--r-- | youtube_dlc/extractor/photobucket.py (renamed from youtube_dl/extractor/photobucket.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/picarto.py (renamed from youtube_dl/extractor/picarto.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/piksel.py (renamed from youtube_dl/extractor/piksel.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pinkbike.py (renamed from youtube_dl/extractor/pinkbike.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pladform.py (renamed from youtube_dl/extractor/pladform.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/platzi.py (renamed from youtube_dl/extractor/platzi.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/playfm.py (renamed from youtube_dl/extractor/playfm.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/playplustv.py (renamed from youtube_dl/extractor/playplustv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/plays.py (renamed from youtube_dl/extractor/plays.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/playtvak.py (renamed from youtube_dl/extractor/playtvak.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/playvid.py (renamed from youtube_dl/extractor/playvid.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/playwire.py (renamed from youtube_dl/extractor/playwire.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pluralsight.py (renamed from youtube_dl/extractor/pluralsight.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/podomatic.py (renamed from youtube_dl/extractor/podomatic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pokemon.py (renamed from youtube_dl/extractor/pokemon.py) | 67 | ||||
-rw-r--r-- | youtube_dlc/extractor/polskieradio.py (renamed from youtube_dl/extractor/polskieradio.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/popcorntimes.py (renamed from youtube_dl/extractor/popcorntimes.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/popcorntv.py (renamed from youtube_dl/extractor/popcorntv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/porn91.py (renamed from youtube_dl/extractor/porn91.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/porncom.py (renamed from youtube_dl/extractor/porncom.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pornhd.py (renamed from youtube_dl/extractor/pornhd.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pornhub.py (renamed from youtube_dl/extractor/pornhub.py) | 19 | ||||
-rw-r--r-- | youtube_dlc/extractor/pornotube.py (renamed from youtube_dl/extractor/pornotube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pornovoisines.py (renamed from youtube_dl/extractor/pornovoisines.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pornoxo.py (renamed from youtube_dl/extractor/pornoxo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/presstv.py (renamed from youtube_dl/extractor/presstv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/prosiebensat1.py (renamed from youtube_dl/extractor/prosiebensat1.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/puhutv.py (renamed from youtube_dl/extractor/puhutv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/puls4.py (renamed from youtube_dl/extractor/puls4.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/pyvideo.py (renamed from youtube_dl/extractor/pyvideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/qqmusic.py (renamed from youtube_dl/extractor/qqmusic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/r7.py (renamed from youtube_dl/extractor/r7.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/radiobremen.py (renamed from youtube_dl/extractor/radiobremen.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/radiocanada.py (renamed from youtube_dl/extractor/radiocanada.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/radiode.py (renamed from youtube_dl/extractor/radiode.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/radiofrance.py (renamed from youtube_dl/extractor/radiofrance.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/radiojavan.py (renamed from youtube_dl/extractor/radiojavan.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rai.py (renamed from youtube_dl/extractor/rai.py) | 127 | ||||
-rw-r--r-- | youtube_dlc/extractor/raywenderlich.py (renamed from youtube_dl/extractor/raywenderlich.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rbmaradio.py (renamed from youtube_dl/extractor/rbmaradio.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rds.py (renamed from youtube_dl/extractor/rds.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/redbulltv.py | 229 | ||||
-rw-r--r-- | youtube_dlc/extractor/reddit.py (renamed from youtube_dl/extractor/reddit.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/redtube.py (renamed from youtube_dl/extractor/redtube.py) | 5 | ||||
-rw-r--r-- | youtube_dlc/extractor/regiotv.py (renamed from youtube_dl/extractor/regiotv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rentv.py (renamed from youtube_dl/extractor/rentv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/restudy.py (renamed from youtube_dl/extractor/restudy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/reuters.py (renamed from youtube_dl/extractor/reuters.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/reverbnation.py (renamed from youtube_dl/extractor/reverbnation.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rice.py (renamed from youtube_dl/extractor/rice.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rmcdecouverte.py (renamed from youtube_dl/extractor/rmcdecouverte.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ro220.py (renamed from youtube_dl/extractor/ro220.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rockstargames.py (renamed from youtube_dl/extractor/rockstargames.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/roosterteeth.py (renamed from youtube_dl/extractor/roosterteeth.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rottentomatoes.py (renamed from youtube_dl/extractor/rottentomatoes.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/roxwel.py (renamed from youtube_dl/extractor/roxwel.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rozhlas.py (renamed from youtube_dl/extractor/rozhlas.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rtbf.py (renamed from youtube_dl/extractor/rtbf.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rte.py (renamed from youtube_dl/extractor/rte.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rtl2.py (renamed from youtube_dl/extractor/rtl2.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rtlnl.py (renamed from youtube_dl/extractor/rtlnl.py) | 24 | ||||
-rw-r--r-- | youtube_dlc/extractor/rtp.py (renamed from youtube_dl/extractor/rtp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rts.py (renamed from youtube_dl/extractor/rts.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rtve.py (renamed from youtube_dl/extractor/rtve.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rtvnh.py (renamed from youtube_dl/extractor/rtvnh.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rtvs.py (renamed from youtube_dl/extractor/rtvs.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ruhd.py (renamed from youtube_dl/extractor/ruhd.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rutube.py (renamed from youtube_dl/extractor/rutube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/rutv.py (renamed from youtube_dl/extractor/rutv.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/ruutu.py (renamed from youtube_dl/extractor/ruutu.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ruv.py (renamed from youtube_dl/extractor/ruv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/safari.py (renamed from youtube_dl/extractor/safari.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sapo.py (renamed from youtube_dl/extractor/sapo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/savefrom.py (renamed from youtube_dl/extractor/savefrom.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sbs.py (renamed from youtube_dl/extractor/sbs.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/screencast.py (renamed from youtube_dl/extractor/screencast.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/screencastomatic.py (renamed from youtube_dl/extractor/screencastomatic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/scrippsnetworks.py (renamed from youtube_dl/extractor/scrippsnetworks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/scte.py (renamed from youtube_dl/extractor/scte.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/seeker.py (renamed from youtube_dl/extractor/seeker.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/senateisvp.py (renamed from youtube_dl/extractor/senateisvp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sendtonews.py (renamed from youtube_dl/extractor/sendtonews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/servus.py (renamed from youtube_dl/extractor/servus.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sevenplus.py (renamed from youtube_dl/extractor/sevenplus.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sexu.py (renamed from youtube_dl/extractor/sexu.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/seznamzpravy.py (renamed from youtube_dl/extractor/seznamzpravy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/shahid.py (renamed from youtube_dl/extractor/shahid.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/shared.py (renamed from youtube_dl/extractor/shared.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/showroomlive.py (renamed from youtube_dl/extractor/showroomlive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sina.py (renamed from youtube_dl/extractor/sina.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sixplay.py (renamed from youtube_dl/extractor/sixplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sky.py (renamed from youtube_dl/extractor/sky.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/skylinewebcams.py (renamed from youtube_dl/extractor/skylinewebcams.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/skynewsarabia.py (renamed from youtube_dl/extractor/skynewsarabia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/slideshare.py (renamed from youtube_dl/extractor/slideshare.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/slideslive.py (renamed from youtube_dl/extractor/slideslive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/slutload.py (renamed from youtube_dl/extractor/slutload.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/smotri.py (renamed from youtube_dl/extractor/smotri.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/snotr.py (renamed from youtube_dl/extractor/snotr.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sohu.py (renamed from youtube_dl/extractor/sohu.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/sonyliv.py (renamed from youtube_dl/extractor/sonyliv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/soundcloud.py (renamed from youtube_dl/extractor/soundcloud.py) | 121 | ||||
-rw-r--r-- | youtube_dlc/extractor/soundgasm.py (renamed from youtube_dl/extractor/soundgasm.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/southpark.py (renamed from youtube_dl/extractor/southpark.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/spankbang.py (renamed from youtube_dl/extractor/spankbang.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/spankwire.py (renamed from youtube_dl/extractor/spankwire.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/spiegel.py (renamed from youtube_dl/extractor/spiegel.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/spiegeltv.py (renamed from youtube_dl/extractor/spiegeltv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/spike.py (renamed from youtube_dl/extractor/spike.py) | 16 | ||||
-rw-r--r-- | youtube_dlc/extractor/sport5.py (renamed from youtube_dl/extractor/sport5.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sportbox.py (renamed from youtube_dl/extractor/sportbox.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sportdeutschland.py (renamed from youtube_dl/extractor/sportdeutschland.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/springboardplatform.py (renamed from youtube_dl/extractor/springboardplatform.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sprout.py (renamed from youtube_dl/extractor/sprout.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/srgssr.py (renamed from youtube_dl/extractor/srgssr.py) | 8 | ||||
-rw-r--r-- | youtube_dlc/extractor/srmediathek.py (renamed from youtube_dl/extractor/srmediathek.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/stanfordoc.py (renamed from youtube_dl/extractor/stanfordoc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/steam.py (renamed from youtube_dl/extractor/steam.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/stitcher.py (renamed from youtube_dl/extractor/stitcher.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/storyfire.py | 255 | ||||
-rw-r--r-- | youtube_dlc/extractor/streamable.py (renamed from youtube_dl/extractor/streamable.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/streamcloud.py (renamed from youtube_dl/extractor/streamcloud.py) | 4 | ||||
-rw-r--r-- | youtube_dlc/extractor/streamcz.py (renamed from youtube_dl/extractor/streamcz.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/streetvoice.py (renamed from youtube_dl/extractor/streetvoice.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/stretchinternet.py (renamed from youtube_dl/extractor/stretchinternet.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/stv.py (renamed from youtube_dl/extractor/stv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sunporno.py (renamed from youtube_dl/extractor/sunporno.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sverigesradio.py (renamed from youtube_dl/extractor/sverigesradio.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/svt.py (renamed from youtube_dl/extractor/svt.py) | 4 | ||||
-rw-r--r-- | youtube_dlc/extractor/swrmediathek.py (renamed from youtube_dl/extractor/swrmediathek.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/syfy.py (renamed from youtube_dl/extractor/syfy.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/sztvhu.py (renamed from youtube_dl/extractor/sztvhu.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tagesschau.py (renamed from youtube_dl/extractor/tagesschau.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tass.py (renamed from youtube_dl/extractor/tass.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tastytrade.py (renamed from youtube_dl/extractor/tastytrade.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tbs.py (renamed from youtube_dl/extractor/tbs.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tdslifeway.py (renamed from youtube_dl/extractor/tdslifeway.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/teachable.py (renamed from youtube_dl/extractor/teachable.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/teachertube.py (renamed from youtube_dl/extractor/teachertube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/teachingchannel.py (renamed from youtube_dl/extractor/teachingchannel.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/teamcoco.py (renamed from youtube_dl/extractor/teamcoco.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/teamtreehouse.py (renamed from youtube_dl/extractor/teamtreehouse.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/techtalks.py (renamed from youtube_dl/extractor/techtalks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ted.py (renamed from youtube_dl/extractor/ted.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tele13.py (renamed from youtube_dl/extractor/tele13.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tele5.py (renamed from youtube_dl/extractor/tele5.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/telebruxelles.py (renamed from youtube_dl/extractor/telebruxelles.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/telecinco.py (renamed from youtube_dl/extractor/telecinco.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/telegraaf.py (renamed from youtube_dl/extractor/telegraaf.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/telemb.py (renamed from youtube_dl/extractor/telemb.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/telequebec.py (renamed from youtube_dl/extractor/telequebec.py) | 45 | ||||
-rw-r--r-- | youtube_dlc/extractor/teletask.py (renamed from youtube_dl/extractor/teletask.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/telewebion.py (renamed from youtube_dl/extractor/telewebion.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tennistv.py (renamed from youtube_dl/extractor/tennistv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tenplay.py (renamed from youtube_dl/extractor/tenplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/testurl.py (renamed from youtube_dl/extractor/testurl.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tf1.py (renamed from youtube_dl/extractor/tf1.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tfo.py (renamed from youtube_dl/extractor/tfo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/theintercept.py (renamed from youtube_dl/extractor/theintercept.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/theplatform.py (renamed from youtube_dl/extractor/theplatform.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/thescene.py (renamed from youtube_dl/extractor/thescene.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/thestar.py (renamed from youtube_dl/extractor/thestar.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/thesun.py (renamed from youtube_dl/extractor/thesun.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/theweatherchannel.py (renamed from youtube_dl/extractor/theweatherchannel.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/thisamericanlife.py (renamed from youtube_dl/extractor/thisamericanlife.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/thisav.py (renamed from youtube_dl/extractor/thisav.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/thisoldhouse.py (renamed from youtube_dl/extractor/thisoldhouse.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/threeqsdn.py (renamed from youtube_dl/extractor/threeqsdn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tiktok.py | 139 | ||||
-rw-r--r-- | youtube_dlc/extractor/tinypic.py (renamed from youtube_dl/extractor/tinypic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tmz.py (renamed from youtube_dl/extractor/tmz.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tnaflix.py (renamed from youtube_dl/extractor/tnaflix.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/toggle.py (renamed from youtube_dl/extractor/toggle.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tonline.py (renamed from youtube_dl/extractor/tonline.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/toongoggles.py (renamed from youtube_dl/extractor/toongoggles.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/toutv.py (renamed from youtube_dl/extractor/toutv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/toypics.py (renamed from youtube_dl/extractor/toypics.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/traileraddict.py (renamed from youtube_dl/extractor/traileraddict.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/trilulilu.py (renamed from youtube_dl/extractor/trilulilu.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/trunews.py (renamed from youtube_dl/extractor/trunews.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/trutv.py (renamed from youtube_dl/extractor/trutv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tube8.py (renamed from youtube_dl/extractor/tube8.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tubitv.py (renamed from youtube_dl/extractor/tubitv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tudou.py (renamed from youtube_dl/extractor/tudou.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tumblr.py (renamed from youtube_dl/extractor/tumblr.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tunein.py (renamed from youtube_dl/extractor/tunein.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tunepk.py (renamed from youtube_dl/extractor/tunepk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/turbo.py (renamed from youtube_dl/extractor/turbo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/turner.py (renamed from youtube_dl/extractor/turner.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tv2.py (renamed from youtube_dl/extractor/tv2.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tv2dk.py (renamed from youtube_dl/extractor/tv2dk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tv2hu.py (renamed from youtube_dl/extractor/tv2hu.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tv4.py (renamed from youtube_dl/extractor/tv4.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tv5mondeplus.py (renamed from youtube_dl/extractor/tv5mondeplus.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tva.py (renamed from youtube_dl/extractor/tva.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvanouvelles.py (renamed from youtube_dl/extractor/tvanouvelles.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvc.py (renamed from youtube_dl/extractor/tvc.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvigle.py (renamed from youtube_dl/extractor/tvigle.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvland.py (renamed from youtube_dl/extractor/tvland.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvn24.py (renamed from youtube_dl/extractor/tvn24.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvnet.py (renamed from youtube_dl/extractor/tvnet.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvnoe.py (renamed from youtube_dl/extractor/tvnoe.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvnow.py (renamed from youtube_dl/extractor/tvnow.py) | 158 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvp.py (renamed from youtube_dl/extractor/tvp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvplay.py (renamed from youtube_dl/extractor/tvplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tvplayer.py (renamed from youtube_dl/extractor/tvplayer.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/tweakers.py (renamed from youtube_dl/extractor/tweakers.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/twentyfourvideo.py (renamed from youtube_dl/extractor/twentyfourvideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/twentymin.py (renamed from youtube_dl/extractor/twentymin.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/twentythreevideo.py (renamed from youtube_dl/extractor/twentythreevideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/twitcasting.py (renamed from youtube_dl/extractor/twitcasting.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/twitch.py (renamed from youtube_dl/extractor/twitch.py) | 146 | ||||
-rw-r--r-- | youtube_dlc/extractor/twitter.py (renamed from youtube_dl/extractor/twitter.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/udemy.py (renamed from youtube_dl/extractor/udemy.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/udn.py (renamed from youtube_dl/extractor/udn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ufctv.py (renamed from youtube_dl/extractor/ufctv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/uktvplay.py (renamed from youtube_dl/extractor/uktvplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/umg.py (renamed from youtube_dl/extractor/umg.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/unistra.py (renamed from youtube_dl/extractor/unistra.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/unity.py (renamed from youtube_dl/extractor/unity.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/uol.py (renamed from youtube_dl/extractor/uol.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/uplynk.py (renamed from youtube_dl/extractor/uplynk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/urort.py (renamed from youtube_dl/extractor/urort.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/urplay.py (renamed from youtube_dl/extractor/urplay.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/usanetwork.py (renamed from youtube_dl/extractor/usanetwork.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/usatoday.py (renamed from youtube_dl/extractor/usatoday.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ustream.py (renamed from youtube_dl/extractor/ustream.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ustudio.py (renamed from youtube_dl/extractor/ustudio.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/varzesh3.py (renamed from youtube_dl/extractor/varzesh3.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vbox7.py (renamed from youtube_dl/extractor/vbox7.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/veehd.py (renamed from youtube_dl/extractor/veehd.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/veoh.py (renamed from youtube_dl/extractor/veoh.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vesti.py (renamed from youtube_dl/extractor/vesti.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vevo.py (renamed from youtube_dl/extractor/vevo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vgtv.py (renamed from youtube_dl/extractor/vgtv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vh1.py (renamed from youtube_dl/extractor/vh1.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/vice.py (renamed from youtube_dl/extractor/vice.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vidbit.py (renamed from youtube_dl/extractor/vidbit.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/viddler.py (renamed from youtube_dl/extractor/viddler.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/videa.py (renamed from youtube_dl/extractor/videa.py) | 62 | ||||
-rw-r--r-- | youtube_dlc/extractor/videodetective.py (renamed from youtube_dl/extractor/videodetective.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/videofyme.py (renamed from youtube_dl/extractor/videofyme.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/videomore.py (renamed from youtube_dl/extractor/videomore.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/videopress.py (renamed from youtube_dl/extractor/videopress.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vidio.py (renamed from youtube_dl/extractor/vidio.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vidlii.py (renamed from youtube_dl/extractor/vidlii.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vidme.py (renamed from youtube_dl/extractor/vidme.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vidzi.py (renamed from youtube_dl/extractor/vidzi.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/vier.py (renamed from youtube_dl/extractor/vier.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/viewlift.py (renamed from youtube_dl/extractor/viewlift.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/viidea.py (renamed from youtube_dl/extractor/viidea.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/viki.py (renamed from youtube_dl/extractor/viki.py) | 122 | ||||
-rw-r--r-- | youtube_dlc/extractor/vimeo.py (renamed from youtube_dl/extractor/vimeo.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/extractor/vimple.py (renamed from youtube_dl/extractor/vimple.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vine.py (renamed from youtube_dl/extractor/vine.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/viqeo.py (renamed from youtube_dl/extractor/viqeo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/viu.py (renamed from youtube_dl/extractor/viu.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vk.py (renamed from youtube_dl/extractor/vk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vlive.py (renamed from youtube_dl/extractor/vlive.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vodlocker.py (renamed from youtube_dl/extractor/vodlocker.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vodpl.py (renamed from youtube_dl/extractor/vodpl.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vodplatform.py (renamed from youtube_dl/extractor/vodplatform.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/voicerepublic.py (renamed from youtube_dl/extractor/voicerepublic.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/voot.py (renamed from youtube_dl/extractor/voot.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/voxmedia.py (renamed from youtube_dl/extractor/voxmedia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vrak.py (renamed from youtube_dl/extractor/vrak.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vrt.py (renamed from youtube_dl/extractor/vrt.py) | 8 | ||||
-rw-r--r-- | youtube_dlc/extractor/vrv.py (renamed from youtube_dl/extractor/vrv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vshare.py (renamed from youtube_dl/extractor/vshare.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vube.py (renamed from youtube_dl/extractor/vube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vuclip.py (renamed from youtube_dl/extractor/vuclip.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vvvvid.py (renamed from youtube_dl/extractor/vvvvid.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vyborymos.py (renamed from youtube_dl/extractor/vyborymos.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/vzaar.py (renamed from youtube_dl/extractor/vzaar.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/wakanim.py (renamed from youtube_dl/extractor/wakanim.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/walla.py (renamed from youtube_dl/extractor/walla.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/washingtonpost.py (renamed from youtube_dl/extractor/washingtonpost.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/wat.py (renamed from youtube_dl/extractor/wat.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/watchbox.py (renamed from youtube_dl/extractor/watchbox.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/watchindianporn.py (renamed from youtube_dl/extractor/watchindianporn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/wdr.py (renamed from youtube_dl/extractor/wdr.py) | 17 | ||||
-rw-r--r-- | youtube_dlc/extractor/webcaster.py (renamed from youtube_dl/extractor/webcaster.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/webofstories.py (renamed from youtube_dl/extractor/webofstories.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/weibo.py (renamed from youtube_dl/extractor/weibo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/weiqitv.py (renamed from youtube_dl/extractor/weiqitv.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/wistia.py (renamed from youtube_dl/extractor/wistia.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/worldstarhiphop.py (renamed from youtube_dl/extractor/worldstarhiphop.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/wsj.py (renamed from youtube_dl/extractor/wsj.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/wwe.py (renamed from youtube_dl/extractor/wwe.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xbef.py (renamed from youtube_dl/extractor/xbef.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xboxclips.py (renamed from youtube_dl/extractor/xboxclips.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xfileshare.py (renamed from youtube_dl/extractor/xfileshare.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xhamster.py (renamed from youtube_dl/extractor/xhamster.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xiami.py (renamed from youtube_dl/extractor/xiami.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ximalaya.py (renamed from youtube_dl/extractor/ximalaya.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xminus.py (renamed from youtube_dl/extractor/xminus.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xnxx.py (renamed from youtube_dl/extractor/xnxx.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xstream.py (renamed from youtube_dl/extractor/xstream.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xtube.py (renamed from youtube_dl/extractor/xtube.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xuite.py (renamed from youtube_dl/extractor/xuite.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xvideos.py (renamed from youtube_dl/extractor/xvideos.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/xxxymovies.py (renamed from youtube_dl/extractor/xxxymovies.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yahoo.py (renamed from youtube_dl/extractor/yahoo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yandexdisk.py (renamed from youtube_dl/extractor/yandexdisk.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yandexmusic.py (renamed from youtube_dl/extractor/yandexmusic.py) | 4 | ||||
-rw-r--r-- | youtube_dlc/extractor/yandexvideo.py (renamed from youtube_dl/extractor/yandexvideo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yapfiles.py (renamed from youtube_dl/extractor/yapfiles.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yesjapan.py (renamed from youtube_dl/extractor/yesjapan.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yinyuetai.py (renamed from youtube_dl/extractor/yinyuetai.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/ynet.py (renamed from youtube_dl/extractor/ynet.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/youjizz.py (renamed from youtube_dl/extractor/youjizz.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/youku.py (renamed from youtube_dl/extractor/youku.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/younow.py (renamed from youtube_dl/extractor/younow.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/youporn.py (renamed from youtube_dl/extractor/youporn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yourporn.py (renamed from youtube_dl/extractor/yourporn.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/yourupload.py (renamed from youtube_dl/extractor/yourupload.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/youtube.py (renamed from youtube_dl/extractor/youtube.py) | 181 | ||||
-rw-r--r-- | youtube_dlc/extractor/zapiks.py (renamed from youtube_dl/extractor/zapiks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/zaq1.py (renamed from youtube_dl/extractor/zaq1.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/zattoo.py (renamed from youtube_dl/extractor/zattoo.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/zdf.py (renamed from youtube_dl/extractor/zdf.py) | 14 | ||||
-rw-r--r-- | youtube_dlc/extractor/zingmp3.py (renamed from youtube_dl/extractor/zingmp3.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/extractor/zype.py (renamed from youtube_dl/extractor/zype.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/jsinterp.py (renamed from youtube_dl/jsinterp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/options.py (renamed from youtube_dl/options.py) | 70 | ||||
-rw-r--r-- | youtube_dlc/postprocessor/__init__.py (renamed from youtube_dl/postprocessor/__init__.py) | 2 | ||||
-rw-r--r-- | youtube_dlc/postprocessor/common.py (renamed from youtube_dl/postprocessor/common.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/postprocessor/embedthumbnail.py (renamed from youtube_dl/postprocessor/embedthumbnail.py) | 52 | ||||
-rw-r--r-- | youtube_dlc/postprocessor/execafterdownload.py (renamed from youtube_dl/postprocessor/execafterdownload.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/postprocessor/ffmpeg.py (renamed from youtube_dl/postprocessor/ffmpeg.py) | 32 | ||||
-rw-r--r-- | youtube_dlc/postprocessor/metadatafromtitle.py (renamed from youtube_dl/postprocessor/metadatafromtitle.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/postprocessor/xattrpp.py (renamed from youtube_dl/postprocessor/xattrpp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/socks.py (renamed from youtube_dl/socks.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/swfinterp.py (renamed from youtube_dl/swfinterp.py) | 0 | ||||
-rw-r--r-- | youtube_dlc/update.py (renamed from youtube_dl/update.py) | 14 | ||||
-rw-r--r-- | youtube_dlc/utils.py (renamed from youtube_dl/utils.py) | 24 | ||||
-rw-r--r-- | youtube_dlc/version.py (renamed from youtube_dl/version.py) | 2 |
897 files changed, 4828 insertions, 2547 deletions
diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index f05aa66e6..4202b532f 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -1,7 +1,10 @@ --- name: Broken site support about: Report broken or misfunctioning site -title: '' +title: "[Broken]" +labels: Broken +assignees: '' + --- <!-- @@ -17,8 +20,8 @@ title: '' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.09.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.09.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape. - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates. @@ -26,7 +29,7 @@ Carefully read and work through this check list in order to prevent the most com --> - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2020.09.06** +- [ ] I've verified that I'm running youtube-dlc version **2020.09.16** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -35,13 +38,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log <!-- -Provide the complete verbose output of youtube-dl that clearly demonstrates the problem. -Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: +Provide the complete verbose output of youtube-dlc that clearly demonstrates the problem. +Add the `-v` flag to your command line you run youtube-dlc with (`youtube-dlc -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: [debug] System config: [] [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2020.09.06 + [debug] youtube-dlc version 2020.09.16 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index 29beaf437..c68a5bb0f 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -1,8 +1,10 @@ --- name: Site support request about: Request support for a new site -title: '' -labels: 'site-support-request' +title: "[Site Request]" +labels: Request +assignees: '' + --- <!-- @@ -18,16 +20,16 @@ labels: 'site-support-request' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.09.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.09.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. -- Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights. +- Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dlc does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights. - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2020.09.06** +- [ ] I've verified that I'm running youtube-dlcc version **2020.09.16** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index f96b8d2bb..2605e0069 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -1,7 +1,10 @@ --- name: Site feature request about: Request a new functionality for a site -title: '' +title: "[Site Request]" +labels: Request +assignees: '' + --- <!-- @@ -17,21 +20,21 @@ title: '' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.09.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.09.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2020.09.06** +- [ ] I've verified that I'm running youtube-dlc version **2020.09.16** - [ ] I've searched the bugtracker for similar site feature requests including closed ones ## Description <!-- -Provide an explanation of your site feature request in an arbitrary form. Please make sure the description is worded well enough to be understood, see https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient. Provide any additional information, suggested solution and as much context and examples as possible. +Provide an explanation of your site feature request in an arbitrary form. Please make sure the description is worded well enough to be understood, see https://github.com/ytdl-org/youtube-dlc#is-the-description-of-the-issue-itself-sufficient. Provide any additional information, suggested solution and as much context and examples as possible. --> WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index 3a175aa4d..841d6ae5b 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -2,6 +2,9 @@ name: Bug report about: Report a bug unrelated to any particular site or extractor title: '' +labels: '' +assignees: '' + --- <!-- @@ -17,8 +20,8 @@ title: '' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.09.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.09.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape. - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates. @@ -27,7 +30,7 @@ Carefully read and work through this check list in order to prevent the most com --> - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2020.09.06** +- [ ] I've verified that I'm running youtube-dlc version **2020.09.14** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -37,13 +40,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log <!-- -Provide the complete verbose output of youtube-dl that clearly demonstrates the problem. -Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: +Provide the complete verbose output of youtube-dlc that clearly demonstrates the problem. +Add the `-v` flag to your command line you run youtube-dlc with (`youtube-dlc -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: [debug] System config: [] [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2020.09.06 + [debug] youtube-dlc version 2020.09.14 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} @@ -58,7 +61,7 @@ PASTE VERBOSE LOG HERE ## Description <!-- -Provide an explanation of your issue in an arbitrary form. Please make sure the description is worded well enough to be understood, see https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient. Provide any additional information, suggested solution and as much context and examples as possible. +Provide an explanation of your issue in an arbitrary form. Please make sure the description is worded well enough to be understood, see https://github.com/ytdl-org/youtube-dlc#is-the-description-of-the-issue-itself-sufficient. Provide any additional information, suggested solution and as much context and examples as possible. If work on your issue requires account credentials please provide them or explain how one can obtain them. --> diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index 4977079de..1af6a768d 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -1,8 +1,10 @@ --- name: Feature request about: Request a new functionality unrelated to any particular site or extractor -title: '' -labels: 'request' +title: "[Feature Request]" +labels: Request +assignees: '' + --- <!-- @@ -18,21 +20,21 @@ labels: 'request' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.09.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.09.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2020.09.06** +- [ ] I've verified that I'm running youtube-dlc version **2020.09.14** - [ ] I've searched the bugtracker for similar feature requests including closed ones ## Description <!-- -Provide an explanation of your issue in an arbitrary form. Please make sure the description is worded well enough to be understood, see https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient. Provide any additional information, suggested solution and as much context and examples as possible. +Provide an explanation of your issue in an arbitrary form. Please make sure the description is worded well enough to be understood, see https://github.com/ytdl-org/youtube-dlc#is-the-description-of-the-issue-itself-sufficient. Provide any additional information, suggested solution and as much context and examples as possible. --> WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/6_question.md b/.github/ISSUE_TEMPLATE/6_question.md index 1fd7cd5dc..034a9c5ac 100644 --- a/.github/ISSUE_TEMPLATE/6_question.md +++ b/.github/ISSUE_TEMPLATE/6_question.md @@ -1,8 +1,10 @@ --- name: Ask question about: Ask youtube-dl related question -title: '' -labels: 'question' +title: "[Question]" +labels: question +assignees: '' + --- <!-- diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md index c7600d5b5..8f9bb2c33 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md @@ -17,8 +17,8 @@ title: '' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape. - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates. @@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com --> - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've verified that I'm running youtube-dlc version **%(version)s** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -35,13 +35,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log <!-- -Provide the complete verbose output of youtube-dl that clearly demonstrates the problem. -Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: +Provide the complete verbose output of youtube-dlc that clearly demonstrates the problem. +Add the `-v` flag to your command line you run youtube-dlc with (`youtube-dlc -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: [debug] System config: [] [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version %(version)s + [debug] youtube-dlc version %(version)s [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md index d4988e639..9748afd4d 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md @@ -18,16 +18,16 @@ labels: 'site-support-request' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. -- Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights. +- Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dlc does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights. - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've verified that I'm running youtube-dlc version **%(version)s** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md index 65f0a32f3..f274e8aeb 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md @@ -17,14 +17,14 @@ title: '' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've verified that I'm running youtube-dlc version **%(version)s** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md index 41fb14b72..788f1c9a1 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md @@ -17,8 +17,8 @@ title: '' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser. - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape. - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates. @@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com --> - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've verified that I'm running youtube-dlc version **%(version)s** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -37,13 +37,13 @@ Carefully read and work through this check list in order to prevent the most com ## Verbose log <!-- -Provide the complete verbose output of youtube-dl that clearly demonstrates the problem. -Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: +Provide the complete verbose output of youtube-dlc that clearly demonstrates the problem. +Add the `-v` flag to your command line you run youtube-dlc with (`youtube-dlc -v <your command line>`), copy the WHOLE output and insert it below. It should look similar to this: [debug] System config: [] [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version %(version)s + [debug] youtube-dlc version %(version)s [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md index b3431a7f0..9b3b8c3bf 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md @@ -18,14 +18,14 @@ labels: 'request' ## Checklist <!-- -Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl: -- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. +Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc: +- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is %(version)s. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED. - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates. - Finally, put x into all relevant boxes (like this [x]) --> - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've verified that I'm running youtube-dlc version **%(version)s** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..8db7e92f2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,151 @@ +name: Build + +on: + push: + branches: + - release + +jobs: + build_unix: + + runs-on: ubuntu-latest + + outputs: + ytdlc_version: ${{ steps.bump_version.outputs.ytdlc_version }} + upload_url: ${{ steps.create_release.outputs.upload_url }} + sha2_unix: ${{ steps.sha2_file.outputs.sha2_unix }} + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install packages + run: sudo apt-get -y install zip pandoc man + - name: Bump version + id: bump_version + run: python scripts/update-version-workflow.py + - name: Check the output from My action + run: echo "${{ steps.bump_version.outputs.ytdlc_version }}" + - name: Run Make + run: make + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.bump_version.outputs.ytdlc_version }} + release_name: youtube-dlc ${{ steps.bump_version.outputs.ytdlc_version }} + body: | + Changelog: + PLACEHOLDER + draft: false + prerelease: false + - name: Upload youtube-dlc Unix binary + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./youtube-dlc + asset_name: youtube-dlc + asset_content_type: application/octet-stream + - name: Get SHA2-256SUMS for youtube-dlc + id: sha2_file + env: + SHA2: ${{ hashFiles('youtube-dlc') }} + run: echo "::set-output name=sha2_unix::${env:SHA2}" + - name: Install dependencies for pypi + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + rm -rf dist/* + python setup.py sdist bdist_wheel + twine upload dist/* + + build_windows: + + runs-on: windows-latest + + needs: build_unix + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install Requirements + run: pip install pyinstaller + - name: Bump version + run: python scripts/update-version-workflow.py + - name: Run PyInstaller Script + run: python pyinst.py + - name: Upload youtube-dlc.exe Windows binary + id: upload-release-windows + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./dist/youtube-dlc.exe + asset_name: youtube-dlc.exe + asset_content_type: application/octet-stream + - name: Get SHA2-256SUMS for youtube-dlc.exe + id: sha2_file_win + env: + SHA2: ${{ hashFiles('dist/youtube-dlc.exe') }} + run: echo "::set-output name=sha2_windows::${env:SHA2}" + + build_windows32: + + runs-on: windows-latest + + needs: build_unix + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.5.4 32-Bit + uses: actions/setup-python@v2 + with: + python-version: '3.5.4' + architecture: 'x86' + - name: Install Requirements for 32 Bit + run: pip install pyinstaller==3.5 + - name: Bump version + run: python scripts/update-version-workflow.py + - name: Run PyInstaller Script for 32 Bit + run: python pyinst32.py + - name: Upload Executable youtube-dlc_x86.exe + id: upload-release-windows32 + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./dist/youtube-dlc_x86.exe + asset_name: youtube-dlc_x86.exe + asset_content_type: application/octet-stream + - name: Get SHA2-256SUMS for youtube-dlc_x86.exe + id: sha2_file_win32 + env: + SHA2: ${{ hashFiles('dist/youtube-dlc_x86.exe') }} + run: echo "::set-output name=sha2_windows32::${env:SHA2}" + - name: Make SHA2-256SUMS file + env: + SHA2_WINDOWS: ${{ needs.build_windows.outputs.sha2_windows }} + SHA2_WINDOWS32: ${{ steps.sha2_file_win32.outputs.sha2_windows32 }} + SHA2_UNIX: ${{ needs.build_unix.outputs.sha2_unix }} + YTDLC_VERSION: ${{ needs.build_unix.outputs.ytdlc_version }} + run: | + echo "$SHA2_WINDOWS youtube-dlc.exe" > SHA2-256SUMS + echo "$SHA2_WINDOWS32 youtube-dlc32.exe" > SHA2-256SUMS + echo "$SHA2_UNIX youtube-dlc" >> SHA2-256SUMS diff --git a/.github/workflows/python-publish.yml.disable b/.github/workflows/python-publish.yml.disable new file mode 100644 index 000000000..224a00230 --- /dev/null +++ b/.github/workflows/python-publish.yml.disable @@ -0,0 +1,33 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + push: + branches: + - release + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + rm -rf dist/* + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.gitignore b/.gitignore index c4870a6ba..065a14f49 100644 --- a/.gitignore +++ b/.gitignore @@ -11,12 +11,20 @@ dist/ MANIFEST README.txt youtube-dl.1 +youtube-dlc.1 youtube-dl.bash-completion +youtube-dlc.bash-completion youtube-dl.fish +youtube-dlc.fish youtube_dl/extractor/lazy_extractors.py +youtube_dlc/extractor/lazy_extractors.py youtube-dl +youtube-dlc youtube-dl.exe +youtube-dlc.exe youtube-dl.tar.gz +youtube-dlc.tar.gz +youtube-dlc.spec .coverage cover/ updates_key.pem @@ -41,6 +49,7 @@ updates_key.pem test/local_parameters.json .tox youtube-dl.zsh +youtube-dlc.zsh # IntelliJ related files .idea @@ -51,3 +60,5 @@ venv/ # VS Code related files .vscode + +cookies.txt diff --git a/.travis.yml b/.travis.yml index 51afd469a..fb499845e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,29 +12,18 @@ python: dist: trusty env: - YTDL_TEST_SET=core - - YTDL_TEST_SET=download jobs: include: - python: 3.7 dist: xenial env: YTDL_TEST_SET=core - - python: 3.7 - dist: xenial - env: YTDL_TEST_SET=download - python: 3.8 dist: xenial env: YTDL_TEST_SET=core - - python: 3.8 - dist: xenial - env: YTDL_TEST_SET=download - python: 3.8-dev dist: xenial env: YTDL_TEST_SET=core - - python: 3.8-dev - dist: xenial - env: YTDL_TEST_SET=download - env: JYTHON=true; YTDL_TEST_SET=core - - env: JYTHON=true; YTDL_TEST_SET=download - name: flake8 python: 3.8 dist: xenial @@ -44,7 +33,6 @@ jobs: allow_failures: - env: YTDL_TEST_SET=download - env: JYTHON=true; YTDL_TEST_SET=core - - env: JYTHON=true; YTDL_TEST_SET=download before_install: - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi script: ./devscripts/run_tests.sh @@ -1,3 +1,42 @@ +version 2020.09.20 + +Core +* [extractor/common] Relax interaction count extraction in _json_ld ++ [extractor/common] Extract author as uploader for VideoObject in _json_ld +* [downloader/hls] Fix incorrect end byte in Range HTTP header for + media segments with EXT-X-BYTERANGE (#14748, #24512) +* [extractor/common] Handle ssl.CertificateError in _request_webpage (#26601) +* [downloader/http] Improve timeout detection when reading block of data + (#10935) +* [downloader/http] Retry download when urlopen times out (#10935, #26603) + +Extractors +* [redtube] Extend URL regular expression (#26506) +* [twitch] Refactor +* [twitch:stream] Switch to GraphQL and fix reruns (#26535) ++ [telequebec] Add support for brightcove videos (#25833) +* [pornhub] Extract metadata from JSON-LD (#26614) +* [pornhub] Fix view count extraction (#26621, #26614) + + +version 2020.09.14 + +Core ++ [postprocessor/embedthumbnail] Add support for non jpg/png thumbnails + (#25687, #25717) + +Extractors +* [rtlnl] Extend URL regular expression (#26549, #25821) +* [youtube] Fix empty description extraction (#26575, #26006) +* [srgssr] Extend URL regular expression (#26555, #26556, #26578) +* [googledrive] Use redirect URLs for source format (#18877, #23919, #24689, + #26565) +* [svtplay] Fix id extraction (#26576) +* [redbulltv] Improve support for rebull.com TV localized URLs (#22063) ++ [redbulltv] Add support for new redbull.com TV URLs (#22037, #22063) +* [soundcloud:pagedplaylist] Reduce pagination limit (#26557) + + version 2020.09.06 Core diff --git a/MANIFEST.in b/MANIFEST.in index 4e43e99f3..d2cce9a1c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,8 +2,8 @@ include README.md include LICENSE include AUTHORS include ChangeLog -include youtube-dl.bash-completion -include youtube-dl.fish -include youtube-dl.1 +include youtube-dlc.bash-completion +include youtube-dlc.fish +include youtube-dlc.1 recursive-include docs Makefile conf.py *.rst recursive-include test * @@ -1,7 +1,7 @@ -all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites +all: youtube-dlc README.md CONTRIBUTING.md README.txt youtube-dlc.1 youtube-dlc.bash-completion youtube-dlc.zsh youtube-dlc.fish supportedsites clean: - rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe + rm -rf youtube-dlc.1.temp.md youtube-dlc.1 youtube-dlc.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dlc.tar.gz youtube-dlc.zsh youtube-dlc.fish youtube_dlc/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp youtube-dlc youtube-dlc.exe find . -name "*.pyc" -delete find . -name "*.class" -delete @@ -17,23 +17,23 @@ SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then ech # set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2 MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi) -install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish +install: youtube-dlc youtube-dlc.1 youtube-dlc.bash-completion youtube-dlc.zsh youtube-dlc.fish install -d $(DESTDIR)$(BINDIR) - install -m 755 youtube-dl $(DESTDIR)$(BINDIR) + install -m 755 youtube-dlc $(DESTDIR)$(BINDIR) install -d $(DESTDIR)$(MANDIR)/man1 - install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1 + install -m 644 youtube-dlc.1 $(DESTDIR)$(MANDIR)/man1 install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d - install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl + install -m 644 youtube-dlc.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dlc install -d $(DESTDIR)$(SHAREDIR)/zsh/site-functions - install -m 644 youtube-dl.zsh $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_youtube-dl + install -m 644 youtube-dlc.zsh $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_youtube-dlc install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions - install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish + install -m 644 youtube-dlc.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dlc.fish codetest: flake8 . test: - #nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test + #nosetests --with-coverage --cover-package=youtube_dlc --cover-html --verbose --processes 4 test nosetests --verbose test $(MAKE) codetest @@ -51,34 +51,34 @@ offlinetest: codetest --exclude test_youtube_lists.py \ --exclude test_youtube_signature.py -tar: youtube-dl.tar.gz +tar: youtube-dlc.tar.gz .PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites -pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish +pypi-files: youtube-dlc.bash-completion README.txt youtube-dlc.1 youtube-dlc.fish -youtube-dl: youtube_dl/*.py youtube_dl/*/*.py +youtube-dlc: youtube_dlc/*.py youtube_dlc/*/*.py mkdir -p zip - for d in youtube_dl youtube_dl/downloader youtube_dl/extractor youtube_dl/postprocessor ; do \ + for d in youtube_dlc youtube_dlc/downloader youtube_dlc/extractor youtube_dlc/postprocessor ; do \ mkdir -p zip/$$d ;\ cp -pPR $$d/*.py zip/$$d/ ;\ done - touch -t 200001010101 zip/youtube_dl/*.py zip/youtube_dl/*/*.py - mv zip/youtube_dl/__main__.py zip/ - cd zip ; zip -q ../youtube-dl youtube_dl/*.py youtube_dl/*/*.py __main__.py + touch -t 200001010101 zip/youtube_dlc/*.py zip/youtube_dlc/*/*.py + mv zip/youtube_dlc/__main__.py zip/ + cd zip ; zip -q ../youtube-dlc youtube_dlc/*.py youtube_dlc/*/*.py __main__.py rm -rf zip - echo '#!$(PYTHON)' > youtube-dl - cat youtube-dl.zip >> youtube-dl - rm youtube-dl.zip - chmod a+x youtube-dl + echo '#!$(PYTHON)' > youtube-dlc + cat youtube-dlc.zip >> youtube-dlc + rm youtube-dlc.zip + chmod a+x youtube-dlc -README.md: youtube_dl/*.py youtube_dl/*/*.py - COLUMNS=80 $(PYTHON) youtube_dl/__main__.py --help | $(PYTHON) devscripts/make_readme.py +README.md: youtube_dlc/*.py youtube_dlc/*/*.py + COLUMNS=80 $(PYTHON) youtube_dlc/__main__.py --help | $(PYTHON) devscripts/make_readme.py CONTRIBUTING.md: README.md $(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md -issuetemplates: devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/1_broken_site.md .github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md .github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md .github/ISSUE_TEMPLATE_tmpl/4_bug_report.md .github/ISSUE_TEMPLATE_tmpl/5_feature_request.md youtube_dl/version.py +issuetemplates: devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/1_broken_site.md .github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md .github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md .github/ISSUE_TEMPLATE_tmpl/4_bug_report.md .github/ISSUE_TEMPLATE_tmpl/5_feature_request.md youtube_dlc/version.py $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/1_broken_site.md .github/ISSUE_TEMPLATE/1_broken_site.md $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md .github/ISSUE_TEMPLATE/2_site_support_request.md $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -91,34 +91,34 @@ supportedsites: README.txt: README.md pandoc -f $(MARKDOWN) -t plain README.md -o README.txt -youtube-dl.1: README.md - $(PYTHON) devscripts/prepare_manpage.py youtube-dl.1.temp.md - pandoc -s -f $(MARKDOWN) -t man youtube-dl.1.temp.md -o youtube-dl.1 - rm -f youtube-dl.1.temp.md +youtube-dlc.1: README.md + $(PYTHON) devscripts/prepare_manpage.py youtube-dlc.1.temp.md + pandoc -s -f $(MARKDOWN) -t man youtube-dlc.1.temp.md -o youtube-dlc.1 + rm -f youtube-dlc.1.temp.md -youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-completion.in +youtube-dlc.bash-completion: youtube_dlc/*.py youtube_dlc/*/*.py devscripts/bash-completion.in $(PYTHON) devscripts/bash-completion.py -bash-completion: youtube-dl.bash-completion +bash-completion: youtube-dlc.bash-completion -youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in +youtube-dlc.zsh: youtube_dlc/*.py youtube_dlc/*/*.py devscripts/zsh-completion.in $(PYTHON) devscripts/zsh-completion.py -zsh-completion: youtube-dl.zsh +zsh-completion: youtube-dlc.zsh -youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in +youtube-dlc.fish: youtube_dlc/*.py youtube_dlc/*/*.py devscripts/fish-completion.in $(PYTHON) devscripts/fish-completion.py -fish-completion: youtube-dl.fish +fish-completion: youtube-dlc.fish -lazy-extractors: youtube_dl/extractor/lazy_extractors.py +lazy-extractors: youtube_dlc/extractor/lazy_extractors.py -_EXTRACTOR_FILES = $(shell find youtube_dl/extractor -iname '*.py' -and -not -iname 'lazy_extractors.py') -youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES) +_EXTRACTOR_FILES = $(shell find youtube_dlc/extractor -iname '*.py' -and -not -iname 'lazy_extractors.py') +youtube_dlc/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES) $(PYTHON) devscripts/make_lazy_extractors.py $@ -youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog AUTHORS - @tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \ +youtube-dlc.tar.gz: youtube-dlc README.md README.txt youtube-dlc.1 youtube-dlc.bash-completion youtube-dlc.zsh youtube-dlc.fish ChangeLog AUTHORS + @tar -czf youtube-dlc.tar.gz --transform "s|^|youtube-dlc/|" --owner 0 --group 0 \ --exclude '*.DS_Store' \ --exclude '*.kate-swp' \ --exclude '*.pyc' \ @@ -128,8 +128,8 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash- --exclude '.git' \ --exclude 'docs/_build' \ -- \ - bin devscripts test youtube_dl docs \ + bin devscripts test youtube_dlc docs \ ChangeLog AUTHORS LICENSE README.md README.txt \ - Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \ - youtube-dl.zsh youtube-dl.fish setup.py setup.cfg \ - youtube-dl + Makefile MANIFEST.in youtube-dlc.1 youtube-dlc.bash-completion \ + youtube-dlc.zsh youtube-dlc.fish setup.py setup.cfg \ + youtube-dlc @@ -1,54 +1,86 @@ -[](https://travis-ci.org/ytdl-org/youtube-dl) +[](https://pypi.org/project/youtube-dlc) +[](https://travis-ci.com/blackjack4494/youtube-dlc) +[](https://pepy.tech/project/youtube-dlc) -youtube-dl - download videos from youtube.com or other video platforms +[](https://gitter.im/youtube-dlc) +[](https://github.com/blackjack4494/youtube-dlc/blob/master/LICENSE) + +youtube-dlc - download videos from youtube.com or other video platforms. + +youtube-dlc is a fork of youtube-dl with the intention of getting features tested by the community merged in the tool faster, since youtube-dl's development seems to be slowing down. (https://github.com/ytdl-org/youtube-dl/issues/26462) - [INSTALLATION](#installation) - [DESCRIPTION](#description) - [OPTIONS](#options) + - [Network Options:](#network-options) + - [Geo Restriction:](#geo-restriction) + - [Video Selection:](#video-selection) + - [Download Options:](#download-options) + - [Filesystem Options:](#filesystem-options) + - [Thumbnail images:](#thumbnail-images) + - [Verbosity / Simulation Options:](#verbosity--simulation-options) + - [Workarounds:](#workarounds) + - [Video Format Options:](#video-format-options) + - [Subtitle Options:](#subtitle-options) + - [Authentication Options:](#authentication-options) + - [Adobe Pass Options:](#adobe-pass-options) + - [Post-processing Options:](#post-processing-options) + - [Extractor Options:](#extractor-options) - [CONFIGURATION](#configuration) + - [Authentication with `.netrc` file](#authentication-with-netrc-file) - [OUTPUT TEMPLATE](#output-template) + - [Output template and Windows batch files](#output-template-and-windows-batch-files) + - [Output template examples](#output-template-examples) - [FORMAT SELECTION](#format-selection) -- [VIDEO SELECTION](#video-selection) -- [FAQ](#faq) -- [DEVELOPER INSTRUCTIONS](#developer-instructions) -- [EMBEDDING YOUTUBE-DL](#embedding-youtube-dl) -- [BUGS](#bugs) -- [COPYRIGHT](#copyright) + - [Format selection examples](#format-selection-examples) +- [VIDEO SELECTION](#video-selection-1) # INSTALLATION -To install it right away for all UNIX users (Linux, macOS, etc.), type: +**All Platforms** +Preferred way using pip: +You may want to use `python3` instead of `python` - sudo curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/local/bin/youtube-dl - sudo chmod a+rx /usr/local/bin/youtube-dl + python -m pip install --upgrade youtube-dlc -If you do not have curl, you can alternatively use a recent wget: +**UNIX** (Linux, macOS, etc.) +Using wget: - sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl - sudo chmod a+rx /usr/local/bin/youtube-dl + sudo wget https://github.com/blackjack4494/youtube-dlc/releases/latest/download/youtube-dlc -O /usr/local/bin/youtube-dlc + sudo chmod a+rx /usr/local/bin/youtube-dlc -Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in any location on their [PATH](https://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\System32` (e.g. **do not** put in `C:\Windows\System32`). +Using curl: -You can also use pip: + sudo curl -L https://github.com/blackjack4494/youtube-dlc/releases/latest/download/youtube-dlc -o /usr/local/bin/youtube-dlc + sudo chmod a+rx /usr/local/bin/youtube-dlc - sudo -H pip install --upgrade youtube-dl - -This command will update youtube-dl if you have already installed it. See the [pypi page](https://pypi.python.org/pypi/youtube_dl) for more information. -macOS users can install youtube-dl with [Homebrew](https://brew.sh/): +**Windows** users can download [youtube-dlc.exe](https://github.com/blackjack4494/youtube-dlc/releases/latest/download/youtube-dlc.exe) (**do not** put in `C:\Windows\System32`!). + +**Compile** +To build the Windows executable yourself (without version info!) + + python -m pip install --upgrade pyinstaller + pyinstaller.exe youtube_dlc\__main__.py --onefile --name youtube-dlc + +Or simply execute the `make_win.bat` if pyinstaller is installed. +There will be a `youtube-dlc.exe` in `/dist` - brew install youtube-dl +New way to build Windows is to use `python pyinst.py` (please use python3 64Bit) +For 32Bit Version use a 32Bit Version of python (3 preferred here as well) and run `python pyinst32.py` -Or with [MacPorts](https://www.macports.org/): +For Unix: +You will need the required build tools +python, make (GNU), pandoc, zip, nosetests +Then simply type this - sudo port install youtube-dl + make -Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://ytdl-org.github.io/youtube-dl/download.html). # DESCRIPTION -**youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like. +**youtube-dlc** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like. - youtube-dl [OPTIONS] URL [URL...] + youtube-dlc [OPTIONS] URL [URL...] # OPTIONS -h, --help Print this help text and exit @@ -69,19 +101,19 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo extractor --default-search PREFIX Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos - from google videos for youtube-dl "large + from google videos for youtube-dlc "large apple". Use the value "auto" to let - youtube-dl guess ("auto_warning" to emit a + youtube-dlc guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching. --ignore-config Do not read configuration files. When given in the global configuration file - /etc/youtube-dl.conf: Do not read the user + /etc/youtube-dlc.conf: Do not read the user configuration in ~/.config/youtube- - dl/config (%APPDATA%/youtube-dl/config.txt - on Windows) + dlc/config (%APPDATA%/youtube- + dlc/config.txt on Windows) --config-location PATH Location of the configuration file; either the path to the config or its containing directory. @@ -238,7 +270,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo filenames -w, --no-overwrites Do not overwrite files -c, --continue Force resume of partially downloaded files. - By default, youtube-dl will resume + By default, youtube-dlc will resume downloads if possible. --no-continue Do not resume partially downloaded files (restart from beginning) @@ -256,11 +288,11 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo option) --cookies FILE File to read cookies from and dump cookie jar in - --cache-dir DIR Location in the filesystem where youtube-dl - can store some downloaded information + --cache-dir DIR Location in the filesystem where youtube- + dlc can store some downloaded information permanently. By default - $XDG_CACHE_HOME/youtube-dl or - ~/.cache/youtube-dl . At the moment, only + $XDG_CACHE_HOME/youtube-dlc or + ~/.cache/youtube-dlc . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change. @@ -306,8 +338,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo files in the current directory to debug problems --print-traffic Display sent and read HTTP traffic - -C, --call-home Contact the youtube-dl server for debugging - --no-call-home Do NOT contact the youtube-dl server for + -C, --call-home Contact the youtube-dlc server for + debugging + --no-call-home Do NOT contact the youtube-dlc server for debugging ## Workarounds: @@ -346,6 +379,8 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo videos --youtube-skip-dash-manifest Do not download the DASH manifests and related data on YouTube videos + --youtube-skip-hls-manifest Do not download the HLS manifests and + related data on YouTube videos --merge-output-format FORMAT If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, @@ -368,7 +403,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo ## Authentication Options: -u, --username USERNAME Login with this account ID -p, --password PASSWORD Account password. If this option is left - out, youtube-dl will ask interactively. + out, youtube-dlc will ask interactively. -2, --twofactor TWOFACTOR Two-factor authentication code -n, --netrc Use .netrc authentication data --video-password PASSWORD Video password (vimeo, smotri, youku) @@ -379,8 +414,8 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo a list of available MSOs --ap-username USERNAME Multiple-system operator account login --ap-password PASSWORD Multiple-system operator account password. - If this option is left out, youtube-dl will - ask interactively. + If this option is left out, youtube-dlc + will ask interactively. --ap-list-mso List all supported multiple-system operators @@ -396,6 +431,10 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5) + --remux-video FORMAT Remux the video to another container format + if necessary (currently supported: mp4|mkv, + target container format must support video + / audio encoding, remuxing may fail) --recode-video FORMAT Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi) @@ -440,11 +479,14 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo --convert-subs FORMAT Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc) +## Extractor Options: + --ignore-dynamic-mpd Do not process dynamic DASH manifests + # CONFIGURATION -You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and macOS, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself. +You can configure youtube-dlc by placing any supported command line option to a configuration file. On Linux and macOS, the system wide configuration file is located at `/etc/youtube-dlc.conf` and the user wide configuration file at `~/.config/youtube-dlc/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dlc\config.txt` or `C:\Users\<user name>\youtube-dlc.conf`. Note that by default configuration file may not exist so you may need to create it yourself. -For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory: +For example, with the following configuration file youtube-dlc will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory: ``` # Lines starting with # are comments @@ -463,13 +505,13 @@ For example, with the following configuration file youtube-dl will always extrac Note that options in configuration file are just the same options aka switches used in regular command line calls thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`. -You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run. +You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dlc run. -You can also use `--config-location` if you want to use custom configuration file for a particular youtube-dl run. +You can also use `--config-location` if you want to use custom configuration file for a particular youtube-dlc run. ### Authentication with `.netrc` file -You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by only you: +You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dlc execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by only you: ``` touch $HOME/.netrc chmod a-rwx,u+rw $HOME/.netrc @@ -483,7 +525,7 @@ For example: machine youtube login myaccount@gmail.com password my_youtube_password machine twitch login my_twitch_account_name password my_twitch_password ``` -To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration). +To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dlc or place it in the [configuration file](#configuration). On Windows you may also need to setup the `%HOME%` environment variable manually. For example: ``` @@ -496,7 +538,7 @@ The `-o` option allows users to indicate a template for the output file names. **tl;dr:** [navigate me to examples](#output-template-examples). -The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations. Allowed names along with sequence type are: +The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dlc -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations. Allowed names along with sequence type are: - `id` (string): Video identifier - `title` (string): Video title @@ -545,7 +587,7 @@ The basic usage is not to set any template arguments when downloading a single f - `extractor` (string): Name of the extractor - `extractor_key` (string): Key name of the extractor - `epoch` (numeric): Unix epoch when creating the file - - `autonumber` (numeric): Five-digit number that will be increased with each download, starting at zero + - `autonumber` (numeric): Number that will be increased with each download, starting at `--autonumber-start` - `playlist` (string): Name or id of the playlist that contains the video - `playlist_index` (numeric): Index of the video in the playlist padded with leading zeros according to the total length of the playlist - `playlist_id` (string): Playlist identifier @@ -584,7 +626,7 @@ Available for the media that is a track or a part of a music album: Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with `NA`. -For example for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj`, this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory. +For example for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `youtube-dlc test video` and id `BaW_jenozKcj`, this will result in a `youtube-dlc test video-BaW_jenozKcj.mp4` file created in the current directory. For numeric sequences you can use numeric related formatting, for example, `%(view_count)05d` will result in a string with view count padded with zeros up to 5 characters, like in `00042`. @@ -605,31 +647,31 @@ If you are using an output template inside a Windows batch file then you must es Note that on Windows you may need to use double quotes instead of single. ```bash -$ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc -youtube-dl test video ''_ä↭𝕐.mp4 # All kinds of weird characters +$ youtube-dlc --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc +youtube-dlc test video ''_ä↭𝕐.mp4 # All kinds of weird characters -$ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc --restrict-filenames -youtube-dl_test_video_.mp4 # A simple file name +$ youtube-dlc --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc --restrict-filenames +youtube-dlc_test_video_.mp4 # A simple file name # Download YouTube playlist videos in separate directory indexed by video order in a playlist -$ youtube-dl -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re +$ youtube-dlc -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re # Download all playlists of YouTube channel/user keeping each playlist in separate directory: -$ youtube-dl -o '%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/user/TheLinuxFoundation/playlists +$ youtube-dlc -o '%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/user/TheLinuxFoundation/playlists # Download Udemy course keeping each chapter in separate directory under MyVideos directory in your home -$ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/ +$ youtube-dlc -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/ # Download entire series season keeping each series and each season in separate directory under C:/MyVideos -$ youtube-dl -o "C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" https://videomore.ru/kino_v_detalayah/5_sezon/367617 +$ youtube-dlc -o "C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" https://videomore.ru/kino_v_detalayah/5_sezon/367617 # Stream the video being downloaded to stdout -$ youtube-dl -o - BaW_jenozKc +$ youtube-dlc -o - BaW_jenozKc ``` # FORMAT SELECTION -By default youtube-dl tries to download the best available quality, i.e. if you want the best quality you **don't need** to pass any special options, youtube-dl will guess it for you by **default**. +By default youtube-dlc tries to download the best available quality, i.e. if you want the best quality you **don't need** to pass any special options, youtube-dlc will guess it for you by **default**. But sometimes you may want to download in a different format, for example when you are on a slow or intermittent connection. The key mechanism for achieving this is so-called *format selection* based on which you can explicitly specify desired format, select formats based on some criterion or criteria, setup precedence and much more. @@ -688,9 +730,9 @@ You can merge the video and audio of two formats into a single file using `-f <v Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`. -Since the end of April 2015 and version 2015.04.26, youtube-dl uses `-f bestvideo+bestaudio/best` as the default format selection (see [#5447](https://github.com/ytdl-org/youtube-dl/issues/5447), [#5456](https://github.com/ytdl-org/youtube-dl/issues/5456)). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed. +Since the end of April 2015 and version 2015.04.26, youtube-dlc uses `-f bestvideo+bestaudio/best` as the default format selection (see [#5447](https://github.com/ytdl-org/youtube-dl/issues/5447), [#5456](https://github.com/ytdl-org/youtube-dl/issues/5456)). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dlc to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dlc still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed. -If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl. +If you want to preserve the old format selection behavior (prior to youtube-dlc 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dlc. #### Format selection examples @@ -698,19 +740,19 @@ Note that on Windows you may need to use double quotes instead of single. ```bash # Download best mp4 format available or any other best if no mp4 available -$ youtube-dl -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best' +$ youtube-dlc -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best' # Download best format available but no better than 480p -$ youtube-dl -f 'bestvideo[height<=480]+bestaudio/best[height<=480]' +$ youtube-dlc -f 'bestvideo[height<=480]+bestaudio/best[height<=480]' # Download best video only format but no bigger than 50 MB -$ youtube-dl -f 'best[filesize<50M]' +$ youtube-dlc -f 'best[filesize<50M]' # Download best format available via direct link over HTTP/HTTPS protocol -$ youtube-dl -f '(bestvideo+bestaudio/best)[protocol^=http]' +$ youtube-dlc -f '(bestvideo+bestaudio/best)[protocol^=http]' # Download the best video format and the best audio format without merging them -$ youtube-dl -f 'bestvideo,bestaudio' -o '%(title)s.f%(format_id)s.%(ext)s' +$ youtube-dlc -f 'bestvideo,bestaudio' -o '%(title)s.f%(format_id)s.%(ext)s' ``` Note that in the last example, an output template is recommended as bestvideo and bestaudio may have the same file name. @@ -726,721 +768,11 @@ Examples: ```bash # Download only the videos uploaded in the last 6 months -$ youtube-dl --dateafter now-6months +$ youtube-dlc --dateafter now-6months # Download only the videos uploaded on January 1, 1970 -$ youtube-dl --date 19700101 +$ youtube-dlc --date 19700101 $ # Download only the videos uploaded in the 200x decade -$ youtube-dl --dateafter 20000101 --datebefore 20091231 -``` - -# FAQ - -### How do I update youtube-dl? - -If you've followed [our manual installation instructions](https://ytdl-org.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`). - -If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update. - -If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to https://yt-dl.org to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum. - -As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like - - sudo apt-get remove -y youtube-dl - -Afterwards, simply follow [our manual installation instructions](https://ytdl-org.github.io/youtube-dl/download.html): - -``` -sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl -sudo chmod a+rx /usr/local/bin/youtube-dl -hash -r -``` - -Again, from then on you'll be able to update with `sudo youtube-dl -U`. - -### youtube-dl is extremely slow to start on Windows - -Add a file exclusion for `youtube-dl.exe` in Windows Defender settings. - -### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists - -YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos. - -If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging people](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update. - -### I'm getting an error when trying to use output template: `error: using output template conflicts with using title, video ID or auto number` - -Make sure you are not using `-o` with any of these options `-t`, `--title`, `--id`, `-A` or `--auto-number` set in command line or in a configuration file. Remove the latter if any. - -### Do I always have to pass `-citw`? - -By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`. - -### Can you please put the `-b` option back? - -Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it. - -### I get HTTP error 402 when trying to download a video. What's this? - -Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/ytdl-org/youtube-dl/issues/154), but at the moment, your best course of action is pointing a web browser to the youtube URL, solving the CAPTCHA, and restart youtube-dl. - -### Do I need any other programs? - -youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option. - -Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](https://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed. - -### I have downloaded a video but how can I play it? - -Once the video is fully downloaded, use any video player, such as [mpv](https://mpv.io/), [vlc](https://www.videolan.org/) or [mplayer](https://www.mplayerhq.hu/). - -### I extracted a video URL with `-g`, but it does not play on another machine / in my web browser. - -It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies and/or HTTP headers. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl. You can also get necessary cookies and HTTP headers from JSON output obtained with `--dump-json`. - -It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule. - -Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well. - -If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn. - -### ERROR: no fmt_url_map or conn information found in video info - -YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl. - -### ERROR: unable to download video - -YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl. - -### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` - -That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell). - -For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command: - -```youtube-dl 'https://www.youtube.com/watch?t=4&v=BaW_jenozKc'``` - -or - -```youtube-dl https://www.youtube.com/watch?t=4\&v=BaW_jenozKc``` - -For Windows you have to use the double quotes: - -```youtube-dl "https://www.youtube.com/watch?t=4&v=BaW_jenozKc"``` - -### ExtractorError: Could not find JS function u'OF' - -In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl. - -### HTTP Error 429: Too Many Requests or 402: Payment Required - -These two error codes indicate that the service is blocking your IP address because of overuse. Usually this is a soft block meaning that you can gain access again after solving CAPTCHA. Just open a browser and solve a CAPTCHA the service suggests you and after that [pass cookies](#how-do-i-pass-cookies-to-youtube-dl) to youtube-dl. Note that if your machine has multiple external IPs then you should also pass exactly the same IP you've used for solving CAPTCHA with [`--source-address`](#network-options). Also you may need to pass a `User-Agent` HTTP header of your browser with [`--user-agent`](#workarounds). - -If this is not the case (no CAPTCHA suggested to solve by the service) then you can contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--source-address` options](#network-options) to select another IP address. - -### SyntaxError: Non-ASCII character - -The error - - File "youtube-dl", line 2 - SyntaxError: Non-ASCII character '\x93' ... - -means you're using an outdated version of Python. Please update to Python 2.6 or 2.7. - -### What is this binary file? Where has the code gone? - -Since June 2012 ([#342](https://github.com/ytdl-org/youtube-dl/issues/342)) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`. - -### The exe throws an error due to missing `MSVCR100.dll` - -To run the exe you need to install first the [Microsoft Visual C++ 2010 Redistributable Package (x86)](https://www.microsoft.com/en-US/download/details.aspx?id=5555). - -### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files? - -If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome. - -To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory. - -From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in. - -### How do I put downloads into a specific folder? - -Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration). - -### How do I download a video starting with a `-`? - -Either prepend `https://www.youtube.com/watch?v=` or separate the ID from the options with `--`: - - youtube-dl -- -wNyEUrxzFU - youtube-dl "https://www.youtube.com/watch?v=-wNyEUrxzFU" - -### How do I pass cookies to youtube-dl? - -Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. - -In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox). - -Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, macOS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format. - -Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare). - -### How do I stream directly to media player? - -You will first need to tell youtube-dl to stream media to stdout with `-o -`, and also tell your media player to read from stdin (it must be capable of this for streaming) and then pipe former to latter. For example, streaming to [vlc](https://www.videolan.org/) can be achieved with: - - youtube-dl -o - "https://www.youtube.com/watch?v=BaW_jenozKcj" | vlc - - -### How do I download only new videos from a playlist? - -Use download-archive feature. With this feature you should initially download the complete playlist with `--download-archive /path/to/download/archive/file.txt` that will record identifiers of all the videos in a special file. Each subsequent run with the same `--download-archive` will download only new videos and skip all videos that have been downloaded before. Note that only successful downloads are recorded in the file. - -For example, at first, - - youtube-dl --download-archive archive.txt "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re" - -will download the complete `PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re` playlist and create a file `archive.txt`. Each subsequent run will only download new videos if any: - - youtube-dl --download-archive archive.txt "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re" - -### Should I add `--hls-prefer-native` into my config? - -When youtube-dl detects an HLS video, it can download it either with the built-in downloader or ffmpeg. Since many HLS streams are slightly invalid and ffmpeg/youtube-dl each handle some invalid cases better than the other, there is an option to switch the downloader if needed. - -When youtube-dl knows that one particular downloader works better for a given website, that downloader will be picked. Otherwise, youtube-dl will pick the best downloader for general compatibility, which at the moment happens to be ffmpeg. This choice may change in future versions of youtube-dl, with improvements of the built-in downloader and/or ffmpeg. - -In particular, the generic extractor (used when your website is not in the [list of supported sites by youtube-dl](https://ytdl-org.github.io/youtube-dl/supportedsites.html) cannot mandate one specific downloader. - -If you put either `--hls-prefer-native` or `--hls-prefer-ffmpeg` into your configuration, a different subset of videos will fail to download correctly. Instead, it is much better to [file an issue](https://yt-dl.org/bug) or a pull request which details why the native or the ffmpeg HLS downloader is a better choice for your use case. - -### Can you add support for this anime video site, or site which shows current movies for free? - -As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl. - -A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization. - -Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content. - -### How can I speed up work on my issue? - -(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do: - -First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel. - -Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues. - -If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so). - -Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`. - -### How can I detect whether a given URL is supported by youtube-dl? - -For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from https://example.com/video/1234567 to https://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug. - -It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor. - -If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program. - -# Why do I need to go through that much red tape when filing bugs? - -Before we had the issue template, despite our extensive [bug reporting instructions](#bugs), about 80% of the issue reports we got were useless, for instance because people used ancient versions hundreds of releases old, because of simple syntactic errors (not in youtube-dl but in general shell usage), because the problem was already reported multiple times before, because people did not actually read an error message, even if it said "please install ffmpeg", because people did not mention the URL they were trying to download and many more simple, easy-to-avoid problems, many of whom were totally unrelated to youtube-dl. - -youtube-dl is an open-source project manned by too few volunteers, so we'd rather spend time fixing bugs where we are certain none of those simple problems apply, and where we can be reasonably confident to be able to reproduce the issue without asking the reporter repeatedly. As such, the output of `youtube-dl -v YOUR_URL_HERE` is really all that's required to file an issue. The issue template also guides you through some basic steps you can do, such as checking that your version of youtube-dl is current. - -# DEVELOPER INSTRUCTIONS - -Most users do not need to build youtube-dl and can [download the builds](https://ytdl-org.github.io/youtube-dl/download.html) or get them from their distribution. - -To run youtube-dl as a developer, you don't need to build anything either. Simply execute - - python -m youtube_dl - -To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work: - - python -m unittest discover - python test/test_download.py - nosetests - -See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases. - -If you want to create a build of youtube-dl yourself, you'll need - -* python -* make (only GNU make is supported) -* pandoc -* zip -* nosetests - -### Adding support for a new site - -If you want to add support for a new site, first of all **make sure** this site is **not dedicated to [copyright infringement](README.md#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. youtube-dl does **not support** such sites thus pull requests adding support for them **will be rejected**. - -After you have ensured this site is distributing its content legally, you can follow this quick list (assuming your service is called `yourextractor`): - -1. [Fork this repository](https://github.com/ytdl-org/youtube-dl/fork) -2. Check out the source code with: - - git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git - -3. Start a new git branch with - - cd youtube-dl - git checkout -b yourextractor - -4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`: - - ```python - # coding: utf-8 - from __future__ import unicode_literals - - from .common import InfoExtractor - - - class YourExtractorIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)' - _TEST = { - 'url': 'https://yourextractor.com/watch/42', - 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', - 'info_dict': { - 'id': '42', - 'ext': 'mp4', - 'title': 'Video title goes here', - 'thumbnail': r're:^https?://.*\.jpg$', - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type (for example int or float) - } - } - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - # TODO more code goes here, for example ... - title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title') - - return { - 'id': video_id, - 'title': title, - 'description': self._og_search_description(webpage), - 'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False), - # TODO more properties (see youtube_dl/extractor/common.py) - } - ``` -5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). -6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. -7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. -8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): - - $ flake8 youtube_dl/extractor/yourextractor.py - -9. Make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+. -10. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this: - - $ git add youtube_dl/extractor/extractors.py - $ git add youtube_dl/extractor/yourextractor.py - $ git commit -m '[yourextractor] Add new extractor' - $ git push origin yourextractor - -11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it. - -In any case, thank you very much for your contributions! - -## youtube-dl coding conventions - -This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code. - -Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hosters out of your control and this layout tends to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize dependency on the source's layout and even to make the code foresee potential future changes and be ready for that. This is important because it will allow the extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with a fix incorporated, all the previous versions become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say, some non rolling release distros may never receive an update at all. - -### Mandatory and optional metafields - -For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by an [information dictionary](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303) or simply *info dict*. Only the following meta fields in the *info dict* are considered mandatory for a successful extraction process by youtube-dl: - - - `id` (media identifier) - - `title` (media title) - - `url` (media download URL) or `formats` - -In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` as mandatory. Thus the aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken. - -[Any field](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L188-L303) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. - -#### Example - -Say you have some source dictionary `meta` that you've fetched as JSON with HTTP request and it has a key `summary`: - -```python -meta = self._download_json(url, video_id) -``` - -Assume at this point `meta`'s layout is: - -```python -{ - ... - "summary": "some fancy summary text", - ... -} -``` - -Assume you want to extract `summary` and put it into the resulting info dict as `description`. Since `description` is an optional meta field you should be ready that this key may be missing from the `meta` dict, so that you should extract it like: - -```python -description = meta.get('summary') # correct -``` - -and not like: - -```python -description = meta['summary'] # incorrect -``` - -The latter will break extraction process with `KeyError` if `summary` disappears from `meta` at some later time but with the former approach extraction will just go ahead with `description` set to `None` which is perfectly fine (remember `None` is equivalent to the absence of data). - -Similarly, you should pass `fatal=False` when extracting optional data from a webpage with `_search_regex`, `_html_search_regex` or similar methods, for instance: - -```python -description = self._search_regex( - r'<span[^>]+id="title"[^>]*>([^<]+)<', - webpage, 'description', fatal=False) -``` - -With `fatal` set to `False` if `_search_regex` fails to extract `description` it will emit a warning and continue extraction. - -You can also pass `default=<some fallback value>`, for example: - -```python -description = self._search_regex( - r'<span[^>]+id="title"[^>]*>([^<]+)<', - webpage, 'description', default=None) -``` - -On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that may or may not be present. - -### Provide fallbacks - -When extracting metadata try to do so from multiple sources. For example if `title` is present in several places, try extracting from at least some of them. This makes it more future-proof in case some of the sources become unavailable. - -#### Example - -Say `meta` from the previous example has a `title` and you are about to extract it. Since `title` is a mandatory meta field you should end up with something like: - -```python -title = meta['title'] -``` - -If `title` disappears from `meta` in future due to some changes on the hoster's side the extraction would fail since `title` is mandatory. That's expected. - -Assume that you have some another source you can extract `title` from, for example `og:title` HTML meta of a `webpage`. In this case you can provide a fallback scenario: - -```python -title = meta.get('title') or self._og_search_title(webpage) -``` - -This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`. - -### Regular expressions - -#### Don't capture groups you don't use - -Capturing group must be an indication that it's used somewhere in the code. Any group that is not used must be non capturing. - -##### Example - -Don't capture id attribute name here since you can't use it for anything anyway. - -Correct: - -```python -r'(?:id|ID)=(?P<id>\d+)' -``` - -Incorrect: -```python -r'(id|ID)=(?P<id>\d+)' -``` - - -#### Make regular expressions relaxed and flexible - -When using regular expressions try to write them fuzzy, relaxed and flexible, skipping insignificant parts that are more likely to change, allowing both single and double quotes for quoted values and so on. - -##### Example - -Say you need to extract `title` from the following HTML code: - -```html -<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">some fancy title</span> -``` - -The code for that task should look similar to: - -```python -title = self._search_regex( - r'<span[^>]+class="title"[^>]*>([^<]+)', webpage, 'title') -``` - -Or even better: - -```python -title = self._search_regex( - r'<span[^>]+class=(["\'])title\1[^>]*>(?P<title>[^<]+)', - webpage, 'title', group='title') -``` - -Note how you tolerate potential changes in the `style` attribute's value or switch from using double quotes to single for `class` attribute: - -The code definitely should not look like: - -```python -title = self._search_regex( - r'<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">(.*?)</span>', - webpage, 'title', group='title') -``` - -### Long lines policy - -There is a soft limit to keep lines of code under 80 characters long. This means it should be respected if possible and if it does not make readability and code maintenance worse. - -For example, you should **never** split long string literals like URLs or some other often copied entities over multiple lines to fit this limit: - -Correct: - -```python -'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4' -``` - -Incorrect: - -```python -'https://www.youtube.com/watch?v=FqZTN594JQw&list=' -'PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4' -``` - -### Inline values - -Extracting variables is acceptable for reducing code duplication and improving readability of complex expressions. However, you should avoid extracting variables used only once and moving them to opposite parts of the extractor file, which makes reading the linear flow difficult. - -#### Example - -Correct: - -```python -title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title') -``` - -Incorrect: - -```python -TITLE_RE = r'<title>([^<]+)</title>' -# ...some lines of code... -title = self._html_search_regex(TITLE_RE, webpage, 'title') -``` - -### Collapse fallbacks - -Multiple fallback values can quickly become unwieldy. Collapse multiple fallback values into a single expression via a list of patterns. - -#### Example - -Good: - -```python -description = self._html_search_meta( - ['og:description', 'description', 'twitter:description'], - webpage, 'description', default=None) -``` - -Unwieldy: - -```python -description = ( - self._og_search_description(webpage, default=None) - or self._html_search_meta('description', webpage, default=None) - or self._html_search_meta('twitter:description', webpage, default=None)) -``` - -Methods supporting list of patterns are: `_search_regex`, `_html_search_regex`, `_og_search_property`, `_html_search_meta`. - -### Trailing parentheses - -Always move trailing parentheses after the last argument. - -#### Example - -Correct: - -```python - lambda x: x['ResultSet']['Result'][0]['VideoUrlSet']['VideoUrl'], - list) -``` - -Incorrect: - -```python - lambda x: x['ResultSet']['Result'][0]['VideoUrlSet']['VideoUrl'], - list, -) -``` - -### Use convenience conversion and parsing functions - -Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py): `int_or_none`, `float_or_none`. Use them for string to number conversions as well. - -Use `url_or_none` for safe URL processing. - -Use `try_get` for safe metadata extraction from parsed JSON. - -Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. - -Explore [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions. - -#### More examples - -##### Safely extract optional description from parsed JSON -```python -description = try_get(response, lambda x: x['result']['video'][0]['summary'], compat_str) -``` - -##### Safely extract more optional metadata -```python -video = try_get(response, lambda x: x['result']['video'][0], dict) or {} -description = video.get('summary') -duration = float_or_none(video.get('durationMs'), scale=1000) -view_count = int_or_none(video.get('views')) -``` - -# EMBEDDING YOUTUBE-DL - -youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/ytdl-org/youtube-dl/issues/new). - -From a Python program, you can embed youtube-dl in a more powerful fashion, like this: - -```python -from __future__ import unicode_literals -import youtube_dl - -ydl_opts = {} -with youtube_dl.YoutubeDL(ydl_opts) as ydl: - ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc']) -``` - -Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/ytdl-org/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312). For a start, if you want to intercept youtube-dl's output, set a `logger` object. - -Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file: - -```python -from __future__ import unicode_literals -import youtube_dl - - -class MyLogger(object): - def debug(self, msg): - pass - - def warning(self, msg): - pass - - def error(self, msg): - print(msg) - - -def my_hook(d): - if d['status'] == 'finished': - print('Done downloading, now converting ...') - - -ydl_opts = { - 'format': 'bestaudio/best', - 'postprocessors': [{ - 'key': 'FFmpegExtractAudio', - 'preferredcodec': 'mp3', - 'preferredquality': '192', - }], - 'logger': MyLogger(), - 'progress_hooks': [my_hook], -} -with youtube_dl.YoutubeDL(ydl_opts) as ydl: - ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc']) -``` - -# BUGS - -Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)). - -**Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this: -``` -$ youtube-dl -v <your command line> -[debug] System config: [] -[debug] User config: [] -[debug] Command-line args: [u'-v', u'https://www.youtube.com/watch?v=BaW_jenozKcj'] -[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 -[debug] youtube-dl version 2015.12.06 -[debug] Git HEAD: 135392e -[debug] Python version 2.6.6 - Windows-2003Server-5.2.3790-SP2 -[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 -[debug] Proxy map: {} -... -``` -**Do not post screenshots of verbose logs; only plain text is acceptable.** - -The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever. - -Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist): - -### Is the description of the issue itself sufficient? - -We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts. - -So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious - -- What the problem is -- How it could be fixed -- How your proposed solution would look like - -If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over. - -For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information. - -If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/). - -**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `https://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `https://www.youtube.com/`) is *not* an example URL. - -### Are you using the latest version? - -Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well. - -### Is the issue already documented? - -Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity. - -### Why are existing options not enough? - -Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem. - -### Is there enough context in your bug report? - -People want to solve problems, and often think they do us a favor by breaking down their larger problems (e.g. wanting to skip already downloaded files) to a specific request (e.g. requesting us to look whether the file exists before downloading the info page). However, what often happens is that they break down the problem into two steps: One simple, and one impossible (or extremely complicated one). - -We are then presented with a very complicated request when the original problem could be solved far easier, e.g. by recording the downloaded video IDs in a separate file. To avoid this, you must include the greater context where it is non-obvious. In particular, every feature request that does not consist of adding support for a new site should contain a use case scenario that explains in what situation the missing feature would be useful. - -### Does the issue involve one problem, and one problem only? - -Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones. - -In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, White house podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service. - -### Is anyone going to need the feature? - -Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them. - -### Is your question about youtube-dl? - -It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different, or even the reporter's own, application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug. - -# COPYRIGHT - -youtube-dl is released into the public domain by the copyright holders. - -This README file was originally written by [Daniel Bolton](https://github.com/dbbolton) and is likewise released into the public domain. +$ youtube-dlc --dateafter 20000101 --datebefore 20091231 +```
\ No newline at end of file diff --git a/bin/youtube-dl b/bin/youtube-dl deleted file mode 100755 index fc3cc8ad8..000000000 --- a/bin/youtube-dl +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -import youtube_dl - -if __name__ == '__main__': - youtube_dl.main() diff --git a/devscripts/bash-completion.in b/devscripts/bash-completion.in index 28bd23727..1bf41f2cc 100644 --- a/devscripts/bash-completion.in +++ b/devscripts/bash-completion.in @@ -1,4 +1,4 @@ -__youtube_dl() +__youtube_dlc() { local cur prev opts fileopts diropts keywords COMPREPLY=() @@ -26,4 +26,4 @@ __youtube_dl() fi } -complete -F __youtube_dl youtube-dl +complete -F __youtube_dlc youtube-dlc diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 3d1391334..d68c9b1cc 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -6,9 +6,9 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +import youtube_dlc -BASH_COMPLETION_FILE = "youtube-dl.bash-completion" +BASH_COMPLETION_FILE = "youtube-dlc.bash-completion" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" @@ -26,5 +26,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = youtube_dlc.parseOpts()[0] build_completion(parser) diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py index 4a4295ba9..62dbd2cb1 100644 --- a/devscripts/buildserver.py +++ b/devscripts/buildserver.py @@ -12,7 +12,7 @@ import traceback import os.path sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__))))) -from youtube_dl.compat import ( +from youtube_dlc.compat import ( compat_input, compat_http_server, compat_str, @@ -325,7 +325,7 @@ class YoutubeDLBuilder(object): authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org'] def __init__(self, **kwargs): - if self.repoName != 'youtube-dl': + if self.repoName != 'youtube-dlc': raise BuildError('Invalid repository "%s"' % self.repoName) if self.user not in self.authorizedUsers: raise HTTPError('Unauthorized user "%s"' % self.user, 401) diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 740f04de0..68a33d823 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -15,8 +15,8 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import gettestcases -from youtube_dl.utils import compat_urllib_parse_urlparse -from youtube_dl.utils import compat_urllib_request +from youtube_dlc.utils import compat_urllib_parse_urlparse +from youtube_dlc.utils import compat_urllib_request if len(sys.argv) > 1: METHOD = 'LIST' diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py index 2ddfa1096..4714d81a6 100644 --- a/devscripts/create-github-release.py +++ b/devscripts/create-github-release.py @@ -12,13 +12,13 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.compat import ( +from youtube_dlc.compat import ( compat_basestring, compat_getpass, compat_print, compat_urllib_request, ) -from youtube_dl.utils import ( +from youtube_dlc.utils import ( make_HTTPS_handler, sanitized_Request, ) @@ -98,7 +98,7 @@ def main(): releaser = GitHubReleaser() new_release = releaser.create_release( - version, name='youtube-dl %s' % version, body=body) + version, name='youtube-dlc %s' % version, body=body) release_id = new_release['id'] for asset in os.listdir(build_path): diff --git a/devscripts/fish-completion.in b/devscripts/fish-completion.in index eb79765da..4f08b6d4a 100644 --- a/devscripts/fish-completion.in +++ b/devscripts/fish-completion.in @@ -2,4 +2,4 @@ {{commands}} -complete --command youtube-dl --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" +complete --command youtube-dlc --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index 51d19dd33..be65f4340 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -7,13 +7,14 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl -from youtube_dl.utils import shell_quote +import youtube_dlc +from youtube_dlc.utils import shell_quote -FISH_COMPLETION_FILE = 'youtube-dl.fish' +FISH_COMPLETION_FILE = 'youtube-dlc.fish' FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in' EXTRA_ARGS = { + 'remux-video': ['--arguments', 'mp4 mkv', '--exclusive'], 'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'], # Options that need a file parameter @@ -30,7 +31,7 @@ def build_completion(opt_parser): for group in opt_parser.option_groups: for option in group.option_list: long_option = option.get_opt_string().strip('-') - complete_cmd = ['complete', '--command', 'youtube-dl', '--long-option', long_option] + complete_cmd = ['complete', '--command', 'youtube-dlc', '--long-option', long_option] if option._short_opts: complete_cmd += ['--short-option', option._short_opts[0].strip('-')] if option.help != optparse.SUPPRESS_HELP: @@ -45,5 +46,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = youtube_dlc.parseOpts()[0] build_completion(parser) diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py index e3df42cc2..c89bb547e 100644 --- a/devscripts/generate_aes_testdata.py +++ b/devscripts/generate_aes_testdata.py @@ -7,8 +7,8 @@ import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.utils import intlist_to_bytes -from youtube_dl.aes import aes_encrypt, key_expansion +from youtube_dlc.utils import intlist_to_bytes +from youtube_dlc.aes import aes_encrypt, key_expansion secret_msg = b'Secret message goes here' diff --git a/devscripts/gh-pages/add-version.py b/devscripts/gh-pages/add-version.py index 867ea0048..04588a5ee 100755 --- a/devscripts/gh-pages/add-version.py +++ b/devscripts/gh-pages/add-version.py @@ -22,9 +22,9 @@ if 'signature' in versions_info: new_version = {} filenames = { - 'bin': 'youtube-dl', - 'exe': 'youtube-dl.exe', - 'tar': 'youtube-dl-%s.tar.gz' % version} + 'bin': 'youtube-dlc', + 'exe': 'youtube-dlc.exe', + 'tar': 'youtube-dlc-%s.tar.gz' % version} build_dir = os.path.join('..', '..', 'build', version) for key, filename in filenames.items(): url = 'https://yt-dl.org/downloads/%s/%s' % (version, filename) diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py index 506a62377..b07f1e830 100755 --- a/devscripts/gh-pages/update-feed.py +++ b/devscripts/gh-pages/update-feed.py @@ -11,24 +11,24 @@ atom_template = textwrap.dedent("""\ <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom"> <link rel="self" href="http://ytdl-org.github.io/youtube-dl/update/releases.atom" /> - <title>youtube-dl releases</title> - <id>https://yt-dl.org/feed/youtube-dl-updates-feed</id> + <title>youtube-dlc releases</title> + <id>https://yt-dl.org/feed/youtube-dlc-updates-feed</id> <updated>@TIMESTAMP@</updated> @ENTRIES@ </feed>""") entry_template = textwrap.dedent(""" <entry> - <id>https://yt-dl.org/feed/youtube-dl-updates-feed/youtube-dl-@VERSION@</id> + <id>https://yt-dl.org/feed/youtube-dlc-updates-feed/youtube-dlc-@VERSION@</id> <title>New version @VERSION@</title> - <link href="http://ytdl-org.github.io/youtube-dl" /> + <link href="http://ytdl-org.github.io/youtube-dlc" /> <content type="xhtml"> <div xmlns="http://www.w3.org/1999/xhtml"> Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a> </div> </content> <author> - <name>The youtube-dl maintainers</name> + <name>The youtube-dlc maintainers</name> </author> <updated>@TIMESTAMP@</updated> </entry> diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index 531c93c70..38acb5d9a 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -5,10 +5,10 @@ import sys import os import textwrap -# We must be able to import youtube_dl +# We must be able to import youtube_dlc sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -import youtube_dl +import youtube_dlc def main(): @@ -16,7 +16,7 @@ def main(): template = tmplf.read() ie_htmls = [] - for ie in youtube_dl.list_extractors(age_limit=None): + for ie in youtube_dlc.list_extractors(age_limit=None): ie_html = '<b>{}</b>'.format(ie.IE_NAME) ie_desc = getattr(ie, 'IE_DESC', None) if ie_desc is False: diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index 226d1a5d6..80426fb0a 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -1,9 +1,9 @@ #!/usr/bin/env python from __future__ import unicode_literals -import io +# import io import optparse -import re +# import re def main(): @@ -12,22 +12,22 @@ def main(): if len(args) != 2: parser.error('Expected an input and an output filename') - infile, outfile = args + +""" infile, outfile = args with io.open(infile, encoding='utf-8') as inf: readme = inf.read() - bug_text = re.search( - r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1) - dev_text = re.search( - r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING YOUTUBE-DL', - readme).group(1) + bug_text = re.search( """ +# r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1) +# dev_text = re.search( +# r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING youtube-dlc', +""" readme).group(1) out = bug_text + dev_text with io.open(outfile, 'w', encoding='utf-8') as outf: - outf.write(out) - + outf.write(out) """ if __name__ == '__main__': main() diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index b7ad23d83..37cb0d4ee 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -16,9 +16,9 @@ def main(): with io.open(infile, encoding='utf-8') as inf: issue_template_tmpl = inf.read() - # Get the version from youtube_dl/version.py without importing the package - exec(compile(open('youtube_dl/version.py').read(), - 'youtube_dl/version.py', 'exec')) + # Get the version from youtube_dlc/version.py without importing the package + exec(compile(open('youtube_dlc/version.py').read(), + 'youtube_dlc/version.py', 'exec')) out = issue_template_tmpl % {'version': locals()['__version__']} diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 0a1762dbc..e6de72b33 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -14,8 +14,8 @@ lazy_extractors_filename = sys.argv[1] if os.path.exists(lazy_extractors_filename): os.remove(lazy_extractors_filename) -from youtube_dl.extractor import _ALL_CLASSES -from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor +from youtube_dlc.extractor import _ALL_CLASSES +from youtube_dlc.extractor.common import InfoExtractor, SearchInfoExtractor with open('devscripts/lazy_load_template.py', 'rt') as f: module_template = f.read() diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 8fbce0796..73f203582 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -14,7 +14,7 @@ with io.open(README_FILE, encoding='utf-8') as f: oldreadme = f.read() header = oldreadme[:oldreadme.index('# OPTIONS')] -footer = oldreadme[oldreadme.index('# CONFIGURATION'):] +# footer = oldreadme[oldreadme.index('# CONFIGURATION'):] options = helptext[helptext.index(' General Options:') + 19:] options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) @@ -23,4 +23,4 @@ options = '# OPTIONS\n' + options + '\n' with io.open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) f.write(options) - f.write(footer) + # f.write(footer) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 764795bc5..0ae6f8aa3 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -7,10 +7,10 @@ import os import sys -# Import youtube_dl +# Import youtube_dlc ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') sys.path.insert(0, ROOT_DIR) -import youtube_dl +import youtube_dlc def main(): @@ -33,7 +33,7 @@ def main(): ie_md += ' (Currently broken)' yield ie_md - ies = sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()) + ies = sorted(youtube_dlc.gen_extractors(), key=lambda i: i.IE_NAME.lower()) out = '# Supported sites\n' + ''.join( ' - ' + md + '\n' for md in gen_ies_md(ies)) diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 76bf873e1..843ade482 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -8,7 +8,7 @@ import re ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) README_FILE = os.path.join(ROOT_DIR, 'README.md') -PREFIX = r'''%YOUTUBE-DL(1) +PREFIX = r'''%youtube-dlc(1) # NAME @@ -16,7 +16,7 @@ youtube\-dl \- download videos from youtube.com or other video platforms # SYNOPSIS -**youtube-dl** \[OPTIONS\] URL [URL...] +**youtube-dlc** \[OPTIONS\] URL [URL...] ''' @@ -33,7 +33,7 @@ def main(): readme = f.read() readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme) - readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) + readme = re.sub(r'\s+youtube-dlc \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) readme = PREFIX + readme readme = filter_options(readme) diff --git a/devscripts/release.sh b/devscripts/release.sh index f2411c927..04cb7fec1 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -53,8 +53,8 @@ fi if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi -useless_files=$(find youtube_dl -type f -not -name '*.py') -if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi +useless_files=$(find youtube_dlc -type f -not -name '*.py') +if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dlc: $useless_files"; exit 1; fi if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi @@ -68,18 +68,18 @@ make clean if $skip_tests ; then echo 'SKIPPING TESTS' else - nosetests --verbose --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1 + nosetests --verbose --with-coverage --cover-package=youtube_dlc --cover-html test --stop || exit 1 fi /bin/echo -e "\n### Changing version in version.py..." -sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py +sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dlc/version.py /bin/echo -e "\n### Changing version in ChangeLog..." sed -i "s/<unreleased>/$version/" ChangeLog -/bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..." +/bin/echo -e "\n### Committing documentation, templates and youtube_dlc/version.py..." make README.md CONTRIBUTING.md issuetemplates supportedsites -git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md youtube_dl/version.py ChangeLog +git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md youtube_dlc/version.py ChangeLog git commit $gpg_sign_commits -m "release $version" /bin/echo -e "\n### Now tagging, signing and pushing..." @@ -94,13 +94,13 @@ git push origin "$version" /bin/echo -e "\n### OK, now it is time to build the binaries..." REV=$(git rev-parse HEAD) -make youtube-dl youtube-dl.tar.gz +make youtube-dlc youtube-dlc.tar.gz read -p "VM running? (y/n) " -n 1 -wget "http://$buildserver/build/ytdl-org/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe +wget "http://$buildserver/build/ytdl-org/youtube-dl/youtube-dlc.exe?rev=$REV" -O youtube-dlc.exe mkdir -p "build/$version" -mv youtube-dl youtube-dl.exe "build/$version" -mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz" -RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz" +mv youtube-dlc youtube-dlc.exe "build/$version" +mv youtube-dlc.tar.gz "build/$version/youtube-dlc-$version.tar.gz" +RELEASE_FILES="youtube-dlc youtube-dlc.exe youtube-dlc-$version.tar.gz" (cd build/$version/ && md5sum $RELEASE_FILES > MD5SUMS) (cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS) (cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS) diff --git a/devscripts/show-downloads-statistics.py b/devscripts/show-downloads-statistics.py index 6c8d1cc2d..ef90a56ab 100644 --- a/devscripts/show-downloads-statistics.py +++ b/devscripts/show-downloads-statistics.py @@ -9,11 +9,11 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.compat import ( +from youtube_dlc.compat import ( compat_print, compat_urllib_request, ) -from youtube_dl.utils import format_bytes +from youtube_dlc.utils import format_bytes def format_size(bytes): @@ -36,9 +36,9 @@ for page in itertools.count(1): asset_name = asset['name'] total_bytes += asset['download_count'] * asset['size'] if all(not re.match(p, asset_name) for p in ( - r'^youtube-dl$', - r'^youtube-dl-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$', - r'^youtube-dl\.exe$')): + r'^youtube-dlc$', + r'^youtube-dlc-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$', + r'^youtube-dlc\.exe$')): continue compat_print( ' %s size: %s downloads: %d' diff --git a/devscripts/zsh-completion.in b/devscripts/zsh-completion.in index b394a1ae7..2317f41df 100644 --- a/devscripts/zsh-completion.in +++ b/devscripts/zsh-completion.in @@ -1,6 +1,6 @@ -#compdef youtube-dl +#compdef youtube-dlc -__youtube_dl() { +__youtube_dlc() { local curcontext="$curcontext" fileopts diropts cur prev typeset -A opt_args fileopts="{{fileopts}}" @@ -16,6 +16,8 @@ __youtube_dl() { _path_files elif [[ ${prev} =~ ${diropts} ]]; then _path_files -/ + elif [[ ${prev} == "--remux-video" ]]; then + _arguments '*: :(mp4 mkv)' elif [[ ${prev} == "--recode-video" ]]; then _arguments '*: :(mp4 flv ogg webm mkv)' else @@ -25,4 +27,4 @@ __youtube_dl() { esac } -__youtube_dl
\ No newline at end of file +__youtube_dlc
\ No newline at end of file diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 60aaf76cc..8b957144f 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -6,9 +6,9 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +import youtube_dlc -ZSH_COMPLETION_FILE = "youtube-dl.zsh" +ZSH_COMPLETION_FILE = "youtube-dlc.zsh" ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" @@ -45,5 +45,5 @@ def build_completion(opt_parser): f.write(template) -parser = youtube_dl.parseOpts()[0] +parser = youtube_dlc.parseOpts()[0] build_completion(parser) diff --git a/docs/Makefile b/docs/Makefile index 712218045..a7159ff45 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -85,17 +85,17 @@ qthelp: @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/youtube-dl.qhcp" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/youtube-dlc.qhcp" @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/youtube-dl.qhc" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/youtube-dlc.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/youtube-dl" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/youtube-dl" + @echo "# mkdir -p $$HOME/.local/share/devhelp/youtube-dlc" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/youtube-dlc" @echo "# devhelp" epub: diff --git a/docs/conf.py b/docs/conf.py index 0aaf1b8fc..fa616ebbb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # coding: utf-8 # -# youtube-dl documentation build configuration file, created by +# youtube-dlc documentation build configuration file, created by # sphinx-quickstart on Fri Mar 14 21:05:43 2014. # # This file is execfile()d with the current directory set to its @@ -14,7 +14,7 @@ import sys import os -# Allows to import youtube_dl +# Allows to import youtube_dlc sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # -- General configuration ------------------------------------------------ @@ -36,7 +36,7 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'youtube-dl' +project = u'youtube-dlc' copyright = u'2014, Ricardo Garcia Gonzalez' # The version info for the project you're documenting, acts as replacement for @@ -44,7 +44,7 @@ copyright = u'2014, Ricardo Garcia Gonzalez' # built documents. # # The short X.Y version. -from youtube_dl.version import __version__ +from youtube_dlc.version import __version__ version = __version__ # The full version, including alpha/beta/rc tags. release = version @@ -68,4 +68,4 @@ html_theme = 'default' html_static_path = ['_static'] # Output file base name for HTML help builder. -htmlhelp_basename = 'youtube-dldoc' +htmlhelp_basename = 'youtube-dlcdoc' diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 000000000..b0f8cad14 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,31 @@ +- Q: How to redirect to another extractor? + - A: + - Most simple using only `url_result` + ``` + # get proper url first if needed. + return self.url_result(url) + ``` + - Using `_request_webpage` and `to_screen` in addition + ``` + urlh = self._request_webpage( + url, id, note='Downloading redirect page') + url = urlh.geturl() + self.to_screen('Following redirect: %s' % url) + return self.url_result(url) + ``` + - Using `return` construction + ``` + return { + '_type': 'url_transparent', + 'url': url, + 'ie_key': ExampleIE.ie_key(), + 'id': id, + } + # Alternative if extractor supports internal uri like kaltura + return { + '_type': 'url_transparent', + 'url': 'kaltura:%s:%s' % (partner_id, kaltura_id), + 'ie_key': KalturaIE.ie_key(), + 'id': id, + } + ``` diff --git a/docs/index.rst b/docs/index.rst index b746ff95b..afa26fef1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,13 +1,13 @@ -Welcome to youtube-dl's documentation! +Welcome to youtube-dlc's documentation! ====================================== -*youtube-dl* is a command-line program to download videos from YouTube.com and more sites. +*youtube-dlc* is a command-line program to download videos from YouTube.com and more sites. It can also be used in Python code. Developer guide --------------- -This section contains information for using *youtube-dl* from Python programs. +This section contains information for using *youtube-dlc* from Python programs. .. toctree:: :maxdepth: 2 diff --git a/docs/module_guide.rst b/docs/module_guide.rst index 03d72882e..6413659cf 100644 --- a/docs/module_guide.rst +++ b/docs/module_guide.rst @@ -1,11 +1,11 @@ -Using the ``youtube_dl`` module +Using the ``youtube_dlc`` module =============================== -When using the ``youtube_dl`` module, you start by creating an instance of :class:`YoutubeDL` and adding all the available extractors: +When using the ``youtube_dlc`` module, you start by creating an instance of :class:`YoutubeDL` and adding all the available extractors: .. code-block:: python - >>> from youtube_dl import YoutubeDL + >>> from youtube_dlc import YoutubeDL >>> ydl = YoutubeDL() >>> ydl.add_default_info_extractors() @@ -22,7 +22,7 @@ You use the :meth:`YoutubeDL.extract_info` method for getting the video informat [youtube] BaW_jenozKc: Downloading video info webpage [youtube] BaW_jenozKc: Extracting video information >>> info['title'] - 'youtube-dl test video "\'/\\ä↭𝕐' + 'youtube-dlc test video "\'/\\ä↭𝕐' >>> info['height'], info['width'] (720, 1280) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 5c4e1d58c..10c12b87a 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -6,7 +6,6 @@ - **23video** - **24video** - **3qsdn**: 3Q SDN - - **3sat** - **4tube** - **56.com** - **5min** @@ -41,6 +40,8 @@ - **AlJazeera** - **Allocine** - **AlphaPorno** + - **Alura** + - **AluraCourse** - **AMCNetworks** - **AmericasTestKitchen** - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl @@ -209,6 +210,7 @@ - **daum.net:user** - **DBTV** - **DctpTv** + - **DeezerAlbum** - **DeezerPlaylist** - **defense.gouv.fr** - **democracynow** @@ -224,6 +226,7 @@ - **Disney** - **dlive:stream** - **dlive:vod** + - **DoodStream** - **Dotsub** - **DouyuShow** - **DouyuTV**: 斗鱼 @@ -234,6 +237,8 @@ - **drtv** - **drtv:live** - **DTube** + - **duboku**: www.duboku.co + - **duboku:list**: www.duboku.co entire series - **Dumpert** - **dvtv**: http://video.aktualne.cz/ - **dw** @@ -251,6 +256,7 @@ - **EllenTube** - **EllenTubePlaylist** - **EllenTubeVideo** + - **Elonet** - **ElPais**: El País - **Embedly** - **EMPFlix** @@ -352,6 +358,7 @@ - **hotstar:playlist** - **Howcast** - **HowStuffWorks** + - **hrfernsehen** - **HRTi** - **HRTiPlaylist** - **Huajiao**: 花椒直播 @@ -454,6 +461,7 @@ - **lynda**: lynda.com videos - **lynda:course**: lynda.com online courses - **m6** + - **MagentaMusik360** - **mailru**: Видео@Mail.Ru - **mailru:music**: Музыка@Mail.Ru - **mailru:music:search**: Музыка@Mail.Ru @@ -524,6 +532,7 @@ - **MySpace:album** - **MySpass** - **Myvi** + - **MyVideoGe** - **MyVidster** - **MyviEmbed** - **MyVisionTV** @@ -672,6 +681,7 @@ - **plus.google**: Google Plus - **podomatic** - **Pokemon** + - **PokemonWatch** - **PolskieRadio** - **PolskieRadioCategory** - **Popcorntimes** @@ -717,6 +727,8 @@ - **RayWenderlichCourse** - **RBMARadio** - **RDS**: RDS.ca + - **RedBull** + - **RedBullEmbed** - **RedBullTV** - **RedBullTVRrnContent** - **Reddit** @@ -835,6 +847,9 @@ - **stanfordoc**: Stanford Open ClassRoom - **Steam** - **Stitcher** + - **StoryFire** + - **StoryFireSeries** + - **StoryFireUser** - **Streamable** - **streamcloud.eu** - **StreamCZ** @@ -895,7 +910,6 @@ - **ThisAV** - **ThisOldHouse** - **TikTok** - - **TikTokUser** - **tinypic**: tinypic.com videos - **TMZ** - **TMZArticle** @@ -940,6 +954,7 @@ - **TVNoe** - **TVNow** - **TVNowAnnual** + - **TVNowFilm** - **TVNowNew** - **TVNowSeason** - **TVNowShow** @@ -1147,7 +1162,7 @@ - **Zaq1** - **Zattoo** - **ZattooLive** - - **ZDF** + - **ZDF-3sat** - **ZDFChannel** - **zingmp3**: mp3.zing.vn - **Zype** diff --git a/make_win.bat b/make_win.bat new file mode 100644 index 000000000..891d517b3 --- /dev/null +++ b/make_win.bat @@ -0,0 +1 @@ +py -m PyInstaller youtube_dlc\__main__.py --onefile --name youtube-dlc --version-file win\ver.txt --icon win\icon\cloud.ico
\ No newline at end of file diff --git a/pyinst.py b/pyinst.py new file mode 100644 index 000000000..199f0734f --- /dev/null +++ b/pyinst.py @@ -0,0 +1,92 @@ +from __future__ import unicode_literals +from PyInstaller.utils.win32.versioninfo import ( + VarStruct, VarFileInfo, StringStruct, StringTable, + StringFileInfo, FixedFileInfo, VSVersionInfo, SetVersion, +) +import PyInstaller.__main__ + +from datetime import datetime + +FILE_DESCRIPTION = 'Media Downloader' + +exec(compile(open('youtube_dlc/version.py').read(), 'youtube_dlc/version.py', 'exec')) + +_LATEST_VERSION = locals()['__version__'] + +_OLD_VERSION = _LATEST_VERSION.rsplit("-", 1) + +if len(_OLD_VERSION) > 0: + old_ver = _OLD_VERSION[0] + +old_rev = '' +if len(_OLD_VERSION) > 1: + old_rev = _OLD_VERSION[1] + +now = datetime.now() +# ver = f'{datetime.today():%Y.%m.%d}' +ver = now.strftime("%Y.%m.%d") +rev = '' + +if old_ver == ver: + if old_rev: + rev = int(old_rev) + 1 + else: + rev = 1 + +_SEPARATOR = '-' + +version = _SEPARATOR.join(filter(None, [ver, str(rev)])) + +print(version) + +version_list = ver.split(".") +_year, _month, _day = [int(value) for value in version_list] +_rev = 0 +if rev: + _rev = rev +_ver_tuple = _year, _month, _day, _rev + +version_file = VSVersionInfo( + ffi=FixedFileInfo( + filevers=_ver_tuple, + prodvers=_ver_tuple, + mask=0x3F, + flags=0x0, + OS=0x4, + fileType=0x1, + subtype=0x0, + date=(0, 0), + ), + kids=[ + StringFileInfo( + [ + StringTable( + "040904B0", + [ + StringStruct("Comments", "Youtube-dlc Command Line Interface."), + StringStruct("CompanyName", "theidel@uni-bremen.de"), + StringStruct("FileDescription", FILE_DESCRIPTION), + StringStruct("FileVersion", version), + StringStruct("InternalName", "youtube-dlc"), + StringStruct( + "LegalCopyright", + "theidel@uni-bremen.de | UNLICENSE", + ), + StringStruct("OriginalFilename", "youtube-dlc.exe"), + StringStruct("ProductName", "Youtube-dlc"), + StringStruct("ProductVersion", version + " | git.io/JUGsM"), + ], + ) + ] + ), + VarFileInfo([VarStruct("Translation", [0, 1200])]) + ] +) + +PyInstaller.__main__.run([ + '--name=youtube-dlc', + '--onefile', + '--icon=win/icon/cloud.ico', + 'youtube_dlc/__main__.py', +]) +SetVersion('dist/youtube-dlc.exe', version_file) diff --git a/pyinst32.py b/pyinst32.py new file mode 100644 index 000000000..ea20a69e5 --- /dev/null +++ b/pyinst32.py @@ -0,0 +1,92 @@ +from __future__ import unicode_literals +from PyInstaller.utils.win32.versioninfo import ( + VarStruct, VarFileInfo, StringStruct, StringTable, + StringFileInfo, FixedFileInfo, VSVersionInfo, SetVersion, +) +import PyInstaller.__main__ + +from datetime import datetime + +FILE_DESCRIPTION = 'Media Downloader 32 Bit Version' + +exec(compile(open('youtube_dlc/version.py').read(), 'youtube_dlc/version.py', 'exec')) + +_LATEST_VERSION = locals()['__version__'] + +_OLD_VERSION = _LATEST_VERSION.rsplit("-", 1) + +if len(_OLD_VERSION) > 0: + old_ver = _OLD_VERSION[0] + +old_rev = '' +if len(_OLD_VERSION) > 1: + old_rev = _OLD_VERSION[1] + +now = datetime.now() +# ver = f'{datetime.today():%Y.%m.%d}' +ver = now.strftime("%Y.%m.%d") +rev = '' + +if old_ver == ver: + if old_rev: + rev = int(old_rev) + 1 + else: + rev = 1 + +_SEPARATOR = '-' + +version = _SEPARATOR.join(filter(None, [ver, str(rev)])) + +print(version) + +version_list = ver.split(".") +_year, _month, _day = [int(value) for value in version_list] +_rev = 0 +if rev: + _rev = rev +_ver_tuple = _year, _month, _day, _rev + +version_file = VSVersionInfo( + ffi=FixedFileInfo( + filevers=_ver_tuple, + prodvers=_ver_tuple, + mask=0x3F, + flags=0x0, + OS=0x4, + fileType=0x1, + subtype=0x0, + date=(0, 0), + ), + kids=[ + StringFileInfo( + [ + StringTable( + "040904B0", + [ + StringStruct("Comments", "Youtube-dlc_x86 Command Line Interface."), + StringStruct("CompanyName", "theidel@uni-bremen.de"), + StringStruct("FileDescription", FILE_DESCRIPTION), + StringStruct("FileVersion", version), + StringStruct("InternalName", "youtube-dlc_x86"), + StringStruct( + "LegalCopyright", + "theidel@uni-bremen.de | UNLICENSE", + ), + StringStruct("OriginalFilename", "youtube-dlc_x86.exe"), + StringStruct("ProductName", "Youtube-dlc_x86"), + StringStruct("ProductVersion", version + "_x86 | git.io/JUGsM"), + ], + ) + ] + ), + VarFileInfo([VarStruct("Translation", [0, 1200])]) + ] +) + +PyInstaller.__main__.run([ + '--name=youtube-dlc_x86', + '--onefile', + '--icon=win/icon/cloud.ico', + 'youtube_dlc/__main__.py', +]) +SetVersion('dist/youtube-dlc_x86.exe', version_file) diff --git a/scripts/update-version-workflow.py b/scripts/update-version-workflow.py new file mode 100644 index 000000000..cebcbf1b4 --- /dev/null +++ b/scripts/update-version-workflow.py @@ -0,0 +1,44 @@ +from __future__ import unicode_literals +from datetime import datetime +# import urllib.request + +# response = urllib.request.urlopen('https://blackjack4494.github.io/youtube-dlc/update/LATEST_VERSION') +# _LATEST_VERSION = response.read().decode('utf-8') + +exec(compile(open('youtube_dlc/version.py').read(), 'youtube_dlc/version.py', 'exec')) + +_LATEST_VERSION = locals()['__version__'] + +_OLD_VERSION = _LATEST_VERSION.rsplit("-", 1) + +if len(_OLD_VERSION) > 0: + old_ver = _OLD_VERSION[0] + +old_rev = '' +if len(_OLD_VERSION) > 1: + old_rev = _OLD_VERSION[1] + +now = datetime.now() +# ver = f'{datetime.today():%Y.%m.%d}' +ver = now.strftime("%Y.%m.%d") +rev = '' + +if old_ver == ver: + if old_rev: + rev = int(old_rev) + 1 + else: + rev = 1 + +_SEPARATOR = '-' + +version = _SEPARATOR.join(filter(None, [ver, str(rev)])) + +print('::set-output name=ytdlc_version::' + version) + +file_version_py = open('youtube_dlc/version.py', 'rt') +data = file_version_py.read() +data = data.replace(locals()['__version__'], version) +file_version_py.close() +file_version_py = open('youtube_dlc/version.py', 'wt') +file_version_py.write(data) +file_version_py.close() diff --git a/scripts/update-version.py b/scripts/update-version.py new file mode 100644 index 000000000..5d779717d --- /dev/null +++ b/scripts/update-version.py @@ -0,0 +1,31 @@ +from __future__ import unicode_literals +from datetime import datetime +import urllib.request + +response = urllib.request.urlopen('https://blackjack4494.github.io/youtube-dlc/update/LATEST_VERSION') + +_LATEST_VERSION = response.read().decode('utf-8') + +_OLD_VERSION = _LATEST_VERSION.rsplit("-", 1) + +if len(_OLD_VERSION) > 0: + old_ver = _OLD_VERSION[0] + +old_rev = '' +if len(_OLD_VERSION) > 1: + old_rev = _OLD_VERSION[1] + +now = datetime.now() +# ver = f'{datetime.today():%Y.%m.%d}' +ver = now.strftime("%Y.%m.%d") +rev = '' + +if old_ver == ver: + if old_rev: + rev = int(old_rev) + 1 + else: + rev = 1 + +_SEPARATOR = '-' + +version = _SEPARATOR.join(filter(None, [ver, str(rev)])) @@ -2,5 +2,5 @@ universal = True [flake8] -exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,devscripts/lazy_load_template.py,devscripts/make_issue_template.py,setup.py,build,.git,venv +exclude = youtube_dlc/extractor/__init__.py,devscripts/buildserver.py,devscripts/lazy_load_template.py,devscripts/make_issue_template.py,setup.py,build,.git,venv ignore = E402,E501,E731,E741,W503 @@ -1,68 +1,27 @@ #!/usr/bin/env python # coding: utf-8 -from __future__ import print_function - +from setuptools import setup, Command, find_packages import os.path import warnings import sys - -try: - from setuptools import setup, Command - setuptools_available = True -except ImportError: - from distutils.core import setup, Command - setuptools_available = False from distutils.spawn import spawn -try: - # This will create an exe that needs Microsoft Visual C++ 2008 - # Redistributable Package - import py2exe -except ImportError: - if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': - print('Cannot import py2exe', file=sys.stderr) - exit(1) - -py2exe_options = { - 'bundle_files': 1, - 'compressed': 1, - 'optimize': 2, - 'dist_dir': '.', - 'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'], -} - -# Get the version from youtube_dl/version.py without importing the package -exec(compile(open('youtube_dl/version.py').read(), - 'youtube_dl/version.py', 'exec')) - -DESCRIPTION = 'YouTube video downloader' -LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites' +# Get the version from youtube_dlc/version.py without importing the package +exec(compile(open('youtube_dlc/version.py').read(), + 'youtube_dlc/version.py', 'exec')) -py2exe_console = [{ - 'script': './youtube_dl/__main__.py', - 'dest_base': 'youtube-dl', - 'version': __version__, - 'description': DESCRIPTION, - 'comments': LONG_DESCRIPTION, - 'product_name': 'youtube-dl', - 'product_version': __version__, -}] - -py2exe_params = { - 'console': py2exe_console, - 'options': {'py2exe': py2exe_options}, - 'zipfile': None -} +DESCRIPTION = 'Media downloader supporting various sites such as youtube' +LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites. Based on a more active community fork.' if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': - params = py2exe_params + print("inv") else: files_spec = [ - ('etc/bash_completion.d', ['youtube-dl.bash-completion']), - ('etc/fish/completions', ['youtube-dl.fish']), - ('share/doc/youtube_dl', ['README.txt']), - ('share/man/man1', ['youtube-dl.1']) + ('etc/bash_completion.d', ['youtube-dlc.bash-completion']), + ('etc/fish/completions', ['youtube-dlc.fish']), + ('share/doc/youtube_dlc', ['README.txt']), + ('share/man/man1', ['youtube-dlc.1']) ] root = os.path.dirname(os.path.abspath(__file__)) data_files = [] @@ -78,10 +37,10 @@ else: params = { 'data_files': data_files, } - if setuptools_available: - params['entry_points'] = {'console_scripts': ['youtube-dl = youtube_dl:main']} - else: - params['scripts'] = ['bin/youtube-dl'] + #if setuptools_available: + params['entry_points'] = {'console_scripts': ['youtube-dlc = youtube_dlc:main']} + #else: + # params['scripts'] = ['bin/youtube-dlc'] class build_lazy_extractors(Command): description = 'Build the extractor lazy loading module' @@ -95,54 +54,50 @@ class build_lazy_extractors(Command): def run(self): spawn( - [sys.executable, 'devscripts/make_lazy_extractors.py', 'youtube_dl/extractor/lazy_extractors.py'], + [sys.executable, 'devscripts/make_lazy_extractors.py', 'youtube_dlc/extractor/lazy_extractors.py'], dry_run=self.dry_run, ) setup( - name='youtube_dl', + name="youtube_dlc", version=__version__, + maintainer="Tom-Oliver Heidel", + maintainer_email="theidel@uni-bremen.de", description=DESCRIPTION, long_description=LONG_DESCRIPTION, - url='https://github.com/ytdl-org/youtube-dl', - author='Ricardo Garcia', - author_email='ytdl@yt-dl.org', - maintainer='Sergey M.', - maintainer_email='dstftw@gmail.com', - license='Unlicense', - packages=[ - 'youtube_dl', - 'youtube_dl.extractor', 'youtube_dl.downloader', - 'youtube_dl.postprocessor'], - - # Provokes warning on most systems (why?!) - # test_suite = 'nose.collector', - # test_requires = ['nosetest'], - + # long_description_content_type="text/markdown", + url="https://github.com/blackjack4494/youtube-dlc", + packages=find_packages(exclude=("youtube_dl",)), + #packages=[ + # 'youtube_dlc', + # 'youtube_dlc.extractor', 'youtube_dlc.downloader', + # 'youtube_dlc.postprocessor'], classifiers=[ - 'Topic :: Multimedia :: Video', - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'License :: Public Domain', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: Implementation', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: IronPython', - 'Programming Language :: Python :: Implementation :: Jython', - 'Programming Language :: Python :: Implementation :: PyPy', + "Topic :: Multimedia :: Video", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: IronPython", + "Programming Language :: Python :: Implementation :: Jython", + "Programming Language :: Python :: Implementation :: PyPy", + "License :: Public Domain", + "Operating System :: OS Independent", ], - - cmdclass={'build_lazy_extractors': build_lazy_extractors}, + python_requires='>=2.6', + + cmdclass={'build_lazy_extractors': build_lazy_extractors}, **params -) +)
\ No newline at end of file diff --git a/test/helper.py b/test/helper.py index e62aab11e..f45818b0f 100644 --- a/test/helper.py +++ b/test/helper.py @@ -10,13 +10,13 @@ import types import ssl import sys -import youtube_dl.extractor -from youtube_dl import YoutubeDL -from youtube_dl.compat import ( +import youtube_dlc.extractor +from youtube_dlc import YoutubeDL +from youtube_dlc.compat import ( compat_os_name, compat_str, ) -from youtube_dl.utils import ( +from youtube_dlc.utils import ( preferredencoding, write_string, ) @@ -90,7 +90,7 @@ class FakeYDL(YoutubeDL): def gettestcases(include_onlymatching=False): - for ie in youtube_dl.extractor.gen_extractors(): + for ie in youtube_dlc.extractor.gen_extractors(): for tc in ie.get_testcases(include_onlymatching): yield tc diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 71f6608fe..bdd01e41a 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -10,10 +10,10 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL, expect_dict, expect_value, http_server_port -from youtube_dl.compat import compat_etree_fromstring, compat_http_server -from youtube_dl.extractor.common import InfoExtractor -from youtube_dl.extractor import YoutubeIE, get_info_extractor -from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError +from youtube_dlc.compat import compat_etree_fromstring, compat_http_server +from youtube_dlc.extractor.common import InfoExtractor +from youtube_dlc.extractor import YoutubeIE, get_info_extractor +from youtube_dlc.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError import threading diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 1e204e551..6d02c2a54 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -12,12 +12,12 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import copy from test.helper import FakeYDL, assertRegexpMatches -from youtube_dl import YoutubeDL -from youtube_dl.compat import compat_str, compat_urllib_error -from youtube_dl.extractor import YoutubeIE -from youtube_dl.extractor.common import InfoExtractor -from youtube_dl.postprocessor.common import PostProcessor -from youtube_dl.utils import ExtractorError, match_filter_func +from youtube_dlc import YoutubeDL +from youtube_dlc.compat import compat_str, compat_urllib_error +from youtube_dlc.extractor import YoutubeIE +from youtube_dlc.extractor.common import InfoExtractor +from youtube_dlc.postprocessor.common import PostProcessor +from youtube_dlc.utils import ExtractorError, match_filter_func TEST_URL = 'http://localhost/sample.mp4' diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py index 05f48bd74..615d8a9d8 100644 --- a/test/test_YoutubeDLCookieJar.py +++ b/test/test_YoutubeDLCookieJar.py @@ -10,7 +10,7 @@ import tempfile import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.utils import YoutubeDLCookieJar +from youtube_dlc.utils import YoutubeDLCookieJar class TestYoutubeDLCookieJar(unittest.TestCase): diff --git a/test/test_aes.py b/test/test_aes.py index cc89fb6ab..ef1e1b189 100644 --- a/test/test_aes.py +++ b/test/test_aes.py @@ -8,8 +8,8 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text -from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes +from youtube_dlc.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text +from youtube_dlc.utils import bytes_to_intlist, intlist_to_bytes import base64 # the encrypted data can be generate with 'devscripts/generate_aes_testdata.py' diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py index 6f5513faa..b73bdd767 100644 --- a/test/test_age_restriction.py +++ b/test/test_age_restriction.py @@ -10,7 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import try_rm -from youtube_dl import YoutubeDL +from youtube_dlc import YoutubeDL def _download_restricted(url, filename, age): diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 81056a999..7b6664cac 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -12,7 +12,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import gettestcases -from youtube_dl.extractor import ( +from youtube_dlc.extractor import ( FacebookIE, gen_extractors, YoutubeIE, @@ -70,7 +70,7 @@ class TestAllURLsMatching(unittest.TestCase): def test_youtube_search_matching(self): self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url']) - self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url']) + self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dlc+test+video&filters=video&lclk=video', ['youtube:search_url']) def test_youtube_extract(self): assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id) diff --git a/test/test_cache.py b/test/test_cache.py index a16160142..1167519d1 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL -from youtube_dl.cache import Cache +from youtube_dlc.cache import Cache def _is_empty(d): diff --git a/test/test_compat.py b/test/test_compat.py index 86ff389fd..8c49a001e 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -10,7 +10,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.compat import ( +from youtube_dlc.compat import ( compat_getenv, compat_setenv, compat_etree_Element, @@ -28,11 +28,11 @@ from youtube_dl.compat import ( class TestCompat(unittest.TestCase): def test_compat_getenv(self): test_str = 'тест' - compat_setenv('YOUTUBE_DL_COMPAT_GETENV', test_str) - self.assertEqual(compat_getenv('YOUTUBE_DL_COMPAT_GETENV'), test_str) + compat_setenv('youtube_dlc_COMPAT_GETENV', test_str) + self.assertEqual(compat_getenv('youtube_dlc_COMPAT_GETENV'), test_str) def test_compat_setenv(self): - test_var = 'YOUTUBE_DL_COMPAT_SETENV' + test_var = 'youtube_dlc_COMPAT_SETENV' test_str = 'тест' compat_setenv(test_var, test_str) compat_getenv(test_var) @@ -46,11 +46,11 @@ class TestCompat(unittest.TestCase): compat_setenv('HOME', old_home or '') def test_all_present(self): - import youtube_dl.compat - all_names = youtube_dl.compat.__all__ + import youtube_dlc.compat + all_names = youtube_dlc.compat.__all__ present_names = set(filter( lambda c: '_' in c and not c.startswith('_'), - dir(youtube_dl.compat))) - set(['unicode_literals']) + dir(youtube_dlc.compat))) - set(['unicode_literals']) self.assertEqual(all_names, sorted(present_names)) def test_compat_urllib_parse_unquote(self): diff --git a/test/test_download.py b/test/test_download.py index ebe820dfc..bcd3b4041 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -24,24 +24,24 @@ import io import json import socket -import youtube_dl.YoutubeDL -from youtube_dl.compat import ( +import youtube_dlc.YoutubeDL +from youtube_dlc.compat import ( compat_http_client, compat_urllib_error, compat_HTTPError, ) -from youtube_dl.utils import ( +from youtube_dlc.utils import ( DownloadError, ExtractorError, format_bytes, UnavailableVideoError, ) -from youtube_dl.extractor import get_info_extractor +from youtube_dlc.extractor import get_info_extractor RETRIES = 3 -class YoutubeDL(youtube_dl.YoutubeDL): +class YoutubeDL(youtube_dlc.YoutubeDL): def __init__(self, *args, **kwargs): self.to_stderr = self.to_screen self.processed_info_dicts = [] @@ -92,7 +92,7 @@ class TestDownload(unittest.TestCase): def generator(test_case, tname): def test_template(self): - ie = youtube_dl.extractor.get_info_extractor(test_case['name'])() + ie = youtube_dlc.extractor.get_info_extractor(test_case['name'])() other_ies = [get_info_extractor(ie_key)() for ie_key in test_case.get('add_ie', [])] is_playlist = any(k.startswith('playlist') for k in test_case) test_cases = test_case.get( diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py index 750472281..c8e28bd3a 100644 --- a/test/test_downloader_http.py +++ b/test/test_downloader_http.py @@ -10,10 +10,10 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import http_server_port, try_rm -from youtube_dl import YoutubeDL -from youtube_dl.compat import compat_http_server -from youtube_dl.downloader.http import HttpFD -from youtube_dl.utils import encodeFilename +from youtube_dlc import YoutubeDL +from youtube_dlc.compat import compat_http_server +from youtube_dlc.downloader.http import HttpFD +from youtube_dlc.utils import encodeFilename import threading TEST_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/test/test_execution.py b/test/test_execution.py index 11661bb68..b18e63d73 100644 --- a/test/test_execution.py +++ b/test/test_execution.py @@ -10,7 +10,7 @@ import os import subprocess sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.utils import encodeArgument +from youtube_dlc.utils import encodeArgument rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -23,18 +23,18 @@ except AttributeError: class TestExecution(unittest.TestCase): def test_import(self): - subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir) + subprocess.check_call([sys.executable, '-c', 'import youtube_dlc'], cwd=rootDir) def test_module_exec(self): if sys.version_info >= (2, 7): # Python 2.6 doesn't support package execution - subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL) + subprocess.check_call([sys.executable, '-m', 'youtube_dlc', '--version'], cwd=rootDir, stdout=_DEV_NULL) def test_main_exec(self): - subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL) + subprocess.check_call([sys.executable, 'youtube_dlc/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL) def test_cmdline_umlauts(self): p = subprocess.Popen( - [sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'], + [sys.executable, 'youtube_dlc/__main__.py', encodeArgument('ä'), '--version'], cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE) _, stderr = p.communicate() self.assertFalse(stderr) diff --git a/test/test_http.py b/test/test_http.py index 3ee0a5dda..55c3c6183 100644 --- a/test/test_http.py +++ b/test/test_http.py @@ -9,8 +9,8 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import http_server_port -from youtube_dl import YoutubeDL -from youtube_dl.compat import compat_http_server, compat_urllib_request +from youtube_dlc import YoutubeDL +from youtube_dlc.compat import compat_http_server, compat_urllib_request import ssl import threading diff --git a/test/test_iqiyi_sdk_interpreter.py b/test/test_iqiyi_sdk_interpreter.py index 789059dbe..303609baa 100644 --- a/test/test_iqiyi_sdk_interpreter.py +++ b/test/test_iqiyi_sdk_interpreter.py @@ -9,7 +9,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL -from youtube_dl.extractor import IqiyiIE +from youtube_dlc.extractor import IqiyiIE class IqiyiIEWithCredentials(IqiyiIE): diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index c24b8ca74..97fc8d5aa 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -8,7 +8,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.jsinterp import JSInterpreter +from youtube_dlc.jsinterp import JSInterpreter class TestJSInterpreter(unittest.TestCase): diff --git a/test/test_netrc.py b/test/test_netrc.py index 7cf3a6a2e..566ba37a6 100644 --- a/test/test_netrc.py +++ b/test/test_netrc.py @@ -7,7 +7,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.extractor import ( +from youtube_dlc.extractor import ( gen_extractors, ) diff --git a/test/test_options.py b/test/test_options.py index 3a25a6ba3..dce253373 100644 --- a/test/test_options.py +++ b/test/test_options.py @@ -8,7 +8,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.options import _hide_login_info +from youtube_dlc.options import _hide_login_info class TestOptions(unittest.TestCase): diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py index 4209d1d9a..6f538a3da 100644 --- a/test/test_postprocessors.py +++ b/test/test_postprocessors.py @@ -8,7 +8,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.postprocessor import MetadataFromTitlePP +from youtube_dlc.postprocessor import MetadataFromTitlePP class TestMetadataFromTitle(unittest.TestCase): diff --git a/test/test_socks.py b/test/test_socks.py index 1e68eb0da..be52e2343 100644 --- a/test/test_socks.py +++ b/test/test_socks.py @@ -15,7 +15,7 @@ from test.helper import ( FakeYDL, get_params, ) -from youtube_dl.compat import ( +from youtube_dlc.compat import ( compat_str, compat_urllib_request, ) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 17aaaf20d..86e20cb4b 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -10,7 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL, md5 -from youtube_dl.extractor import ( +from youtube_dlc.extractor import ( YoutubeIE, DailymotionIE, TEDIE, @@ -64,8 +64,8 @@ class TestYoutubeSubtitles(BaseTestSubtitles): self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() self.assertEqual(len(subtitles.keys()), 13) - self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06') - self.assertEqual(md5(subtitles['it']), '6d752b98c31f1cf8d597050c7a2cb4b5') + self.assertEqual(md5(subtitles['en']), '688dd1ce0981683867e7fe6fde2a224b') + self.assertEqual(md5(subtitles['it']), '31324d30b8430b309f7f5979a504a769') for lang in ['fr', 'de']: self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang) @@ -73,13 +73,13 @@ class TestYoutubeSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['subtitlesformat'] = 'ttml' subtitles = self.getSubtitles() - self.assertEqual(md5(subtitles['en']), 'e306f8c42842f723447d9f63ad65df54') + self.assertEqual(md5(subtitles['en']), 'c97ddf1217390906fa9fbd34901f3da2') def test_youtube_subtitles_vtt_format(self): self.DL.params['writesubtitles'] = True self.DL.params['subtitlesformat'] = 'vtt' subtitles = self.getSubtitles() - self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06') + self.assertEqual(md5(subtitles['en']), 'ae1bd34126571a77aabd4d276b28044d') def test_youtube_automatic_captions(self): self.url = '8YoUxe5ncPo' @@ -88,9 +88,15 @@ class TestYoutubeSubtitles(BaseTestSubtitles): subtitles = self.getSubtitles() self.assertTrue(subtitles['it'] is not None) + def test_youtube_no_automatic_captions(self): + self.url = 'QRS8MkLhQmM' + self.DL.params['writeautomaticsub'] = True + subtitles = self.getSubtitles() + self.assertTrue(not subtitles) + def test_youtube_translated_subtitles(self): # This video has a subtitles track, which can be translated - self.url = 'Ky9eprVWzlI' + self.url = 'i0ZabxXmH4Y' self.DL.params['writeautomaticsub'] = True self.DL.params['subtitleslangs'] = ['it'] subtitles = self.getSubtitles() diff --git a/test/test_swfinterp.py b/test/test_swfinterp.py index 9f18055e6..1a8b353e8 100644 --- a/test/test_swfinterp.py +++ b/test/test_swfinterp.py @@ -14,7 +14,7 @@ import json import re import subprocess -from youtube_dl.swfinterp import SWFInterpreter +from youtube_dlc.swfinterp import SWFInterpreter TEST_DIR = os.path.join( diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py index 6c1b7ec91..e6627a3e5 100644 --- a/test/test_unicode_literals.py +++ b/test/test_unicode_literals.py @@ -15,6 +15,8 @@ IGNORED_FILES = [ 'setup.py', # http://bugs.python.org/issue13943 'conf.py', 'buildserver.py', + 'pyinst.py', + 'pyinst32.py', ] IGNORED_DIRS = [ diff --git a/test/test_update.py b/test/test_update.py index d9c71511d..1b144c43c 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -10,7 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import json -from youtube_dl.update import rsa_verify +from youtube_dlc.update import rsa_verify class TestUpdate(unittest.TestCase): diff --git a/test/test_utils.py b/test/test_utils.py index 962fd8d75..95231200b 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -15,7 +15,7 @@ import io import json import xml.etree.ElementTree -from youtube_dl.utils import ( +from youtube_dlc.utils import ( age_restricted, args_to_str, encode_base_n, @@ -105,7 +105,7 @@ from youtube_dl.utils import ( cli_bool_option, parse_codecs, ) -from youtube_dl.compat import ( +from youtube_dlc.compat import ( compat_chr, compat_etree_fromstring, compat_getenv, @@ -240,12 +240,12 @@ class TestUtil(unittest.TestCase): def env(var): return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var) - compat_setenv('YOUTUBE_DL_EXPATH_PATH', 'expanded') - self.assertEqual(expand_path(env('YOUTUBE_DL_EXPATH_PATH')), 'expanded') + compat_setenv('youtube_dlc_EXPATH_PATH', 'expanded') + self.assertEqual(expand_path(env('youtube_dlc_EXPATH_PATH')), 'expanded') self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME')) self.assertEqual(expand_path('~'), compat_getenv('HOME')) self.assertEqual( - expand_path('~/%s' % env('YOUTUBE_DL_EXPATH_PATH')), + expand_path('~/%s' % env('youtube_dlc_EXPATH_PATH')), '%s/expanded' % compat_getenv('HOME')) def test_prepend_extension(self): @@ -1390,8 +1390,8 @@ Line 1 self.assertEqual(caesar('ebg', 'acegik', -2), 'abc') def test_rot47(self): - self.assertEqual(rot47('youtube-dl'), r'J@FEF36\5=') - self.assertEqual(rot47('YOUTUBE-DL'), r'*~&%&qt\s{') + self.assertEqual(rot47('youtube-dlc'), r'J@FEF36\5=4') + self.assertEqual(rot47('YOUTUBE-DLC'), r'*~&%&qt\s{r') def test_urshift(self): self.assertEqual(urshift(3, 1), 1) diff --git a/test/test_verbose_output.py b/test/test_verbose_output.py index c1465fe8c..462f25e03 100644 --- a/test/test_verbose_output.py +++ b/test/test_verbose_output.py @@ -17,7 +17,7 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_arg(self): outp = subprocess.Popen( [ - sys.executable, 'youtube_dl/__main__.py', '-v', + sys.executable, 'youtube_dlc/__main__.py', '-v', '--username', 'johnsmith@gmail.com', '--password', 'secret', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -30,7 +30,7 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_shortarg(self): outp = subprocess.Popen( [ - sys.executable, 'youtube_dl/__main__.py', '-v', + sys.executable, 'youtube_dlc/__main__.py', '-v', '-u', 'johnsmith@gmail.com', '-p', 'secret', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -43,7 +43,7 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_eq(self): outp = subprocess.Popen( [ - sys.executable, 'youtube_dl/__main__.py', '-v', + sys.executable, 'youtube_dlc/__main__.py', '-v', '--username=johnsmith@gmail.com', '--password=secret', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -56,7 +56,7 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_shortarg_eq(self): outp = subprocess.Popen( [ - sys.executable, 'youtube_dl/__main__.py', '-v', + sys.executable, 'youtube_dlc/__main__.py', '-v', '-u=johnsmith@gmail.com', '-p=secret', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py index 41abdfe3b..d98c96c15 100644 --- a/test/test_write_annotations.py +++ b/test/test_write_annotations.py @@ -15,11 +15,11 @@ import io import xml.etree.ElementTree -import youtube_dl.YoutubeDL -import youtube_dl.extractor +import youtube_dlc.YoutubeDL +import youtube_dlc.extractor -class YoutubeDL(youtube_dl.YoutubeDL): +class YoutubeDL(youtube_dlc.YoutubeDL): def __init__(self, *args, **kwargs): super(YoutubeDL, self).__init__(*args, **kwargs) self.to_stderr = self.to_screen @@ -45,7 +45,7 @@ class TestAnnotations(unittest.TestCase): def test_info_json(self): expected = list(EXPECTED_ANNOTATIONS) # Two annotations could have the same text. - ie = youtube_dl.extractor.YoutubeIE() + ie = youtube_dlc.extractor.YoutubeIE() ydl = YoutubeDL(params) ydl.add_info_extractor(ie) ydl.download([TEST_ID]) diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py index e69c57377..4529d2e84 100644 --- a/test/test_youtube_chapters.py +++ b/test/test_youtube_chapters.py @@ -9,7 +9,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import expect_value -from youtube_dl.extractor import YoutubeIE +from youtube_dlc.extractor import YoutubeIE class TestYoutubeChapters(unittest.TestCase): diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index c4f0abbea..a693963ef 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -10,7 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL -from youtube_dl.extractor import ( +from youtube_dlc.extractor import ( YoutubePlaylistIE, YoutubeIE, ) diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 69df30eda..a54b36198 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -13,8 +13,8 @@ import re import string from test.helper import FakeYDL -from youtube_dl.extractor import YoutubeIE -from youtube_dl.compat import compat_str, compat_urlretrieve +from youtube_dlc.extractor import YoutubeIE +from youtube_dlc.compat import compat_str, compat_urlretrieve _TESTS = [ ( @@ -10,5 +10,5 @@ defaultargs = test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py --exclude test_socks.py -commands = nosetests --verbose {posargs:{[testenv]defaultargs}} # --with-coverage --cover-package=youtube_dl --cover-html +commands = nosetests --verbose {posargs:{[testenv]defaultargs}} # --with-coverage --cover-package=youtube_dlc --cover-html # test.test_download:TestDownload.test_NowVideo diff --git a/win/icon/cloud.ico b/win/icon/cloud.ico Binary files differnew file mode 100644 index 000000000..6d742ce63 --- /dev/null +++ b/win/icon/cloud.ico diff --git a/win/ver.txt b/win/ver.txt new file mode 100644 index 000000000..2d0e1bc25 --- /dev/null +++ b/win/ver.txt @@ -0,0 +1,45 @@ +# UTF-8 +# +# For more details about fixed file info 'ffi' see: +# http://msdn.microsoft.com/en-us/library/ms646997.aspx +VSVersionInfo( + ffi=FixedFileInfo( + # filevers and prodvers should be always a tuple with four items: (1, 2, 3, 4) + # Set not needed items to zero 0. + filevers=(16, 9, 2020, 0), + prodvers=(16, 9, 2020, 0), + # Contains a bitmask that specifies the valid bits 'flags'r + mask=0x3f, + # Contains a bitmask that specifies the Boolean attributes of the file. + flags=0x0, + # The operating system for which this file was designed. + # 0x4 - NT and there is no need to change it. + # OS=0x40004, + OS=0x4, + # The general type of file. + # 0x1 - the file is an application. + fileType=0x1, + # The function of the file. + # 0x0 - the function is not defined for this fileType + subtype=0x0, + # Creation date and time stamp. + date=(0, 0) + ), + kids=[ + StringFileInfo( + [ + StringTable( + u'040904B0', + [StringStruct(u'Comments', u'Youtube-dlc Command Line Interface.'), + StringStruct(u'CompanyName', u'theidel@uni-bremen.de'), + StringStruct(u'FileDescription', u'Media Downloader'), + StringStruct(u'FileVersion', u'16.9.2020.0'), + StringStruct(u'InternalName', u'youtube-dlc'), + StringStruct(u'LegalCopyright', u'theidel@uni-bremen.de | UNLICENSE'), + StringStruct(u'OriginalFilename', u'youtube-dlc.exe'), + StringStruct(u'ProductName', u'Youtube-dlc'), + StringStruct(u'ProductVersion', u'16.9.2020.0 | git.io/JUGsM')]) + ]), + VarFileInfo([VarStruct(u'Translation', [0, 1200])]) + ] +) diff --git a/youtube-dl.plugin.zsh b/youtube-dl.plugin.zsh deleted file mode 100644 index 17ab1341a..000000000 --- a/youtube-dl.plugin.zsh +++ /dev/null @@ -1,24 +0,0 @@ -# This allows the youtube-dl command to be installed in ZSH using antigen. -# Antigen is a bundle manager. It allows you to enhance the functionality of -# your zsh session by installing bundles and themes easily. - -# Antigen documentation: -# http://antigen.sharats.me/ -# https://github.com/zsh-users/antigen - -# Install youtube-dl: -# antigen bundle ytdl-org/youtube-dl -# Bundles installed by antigen are available for use immediately. - -# Update youtube-dl (and all other antigen bundles): -# antigen update - -# The antigen command will download the git repository to a folder and then -# execute an enabling script (this file). The complete process for loading the -# code is documented here: -# https://github.com/zsh-users/antigen#notes-on-writing-plugins - -# This specific script just aliases youtube-dl to the python script that this -# library provides. This requires updating the PYTHONPATH to ensure that the -# full set of code can be located. -alias youtube-dl="PYTHONPATH=$(dirname $0) $(dirname $0)/bin/youtube-dl" diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py deleted file mode 100644 index a38b2683d..000000000 --- a/youtube_dl/extractor/deezer.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import unicode_literals - -import json -import re - -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - int_or_none, - orderedSet, -) - - -class DeezerPlaylistIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?deezer\.com/playlist/(?P<id>[0-9]+)' - _TEST = { - 'url': 'http://www.deezer.com/playlist/176747451', - 'info_dict': { - 'id': '176747451', - 'title': 'Best!', - 'uploader': 'Anonymous', - 'thumbnail': r're:^https?://cdn-images\.deezer\.com/images/cover/.*\.jpg$', - }, - 'playlist_count': 30, - 'skip': 'Only available in .de', - } - - def _real_extract(self, url): - if 'test' not in self._downloader.params: - self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') - - mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') - - webpage = self._download_webpage(url, playlist_id) - geoblocking_msg = self._html_search_regex( - r'<p class="soon-txt">(.*?)</p>', webpage, 'geoblocking message', - default=None) - if geoblocking_msg is not None: - raise ExtractorError( - 'Deezer said: %s' % geoblocking_msg, expected=True) - - data_json = self._search_regex( - (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*</script>', - r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), - webpage, 'data JSON') - data = json.loads(data_json) - - playlist_title = data.get('DATA', {}).get('TITLE') - playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') - playlist_thumbnail = self._search_regex( - r'<img id="naboo_playlist_image".*?src="([^"]+)"', webpage, - 'playlist thumbnail') - - preview_pattern = self._search_regex( - r"var SOUND_PREVIEW_GATEWAY\s*=\s*'([^']+)';", webpage, - 'preview URL pattern', fatal=False) - entries = [] - for s in data['SONGS']['data']: - puid = s['MD5_ORIGIN'] - preview_video_url = preview_pattern.\ - replace('{0}', puid[0]).\ - replace('{1}', puid).\ - replace('{2}', s['MEDIA_VERSION']) - formats = [{ - 'format_id': 'preview', - 'url': preview_video_url, - 'preference': -100, # Only the first 30 seconds - 'ext': 'mp3', - }] - self._sort_formats(formats) - artists = ', '.join( - orderedSet(a['ART_NAME'] for a in s['ARTISTS'])) - entries.append({ - 'id': s['SNG_ID'], - 'duration': int_or_none(s.get('DURATION')), - 'title': '%s - %s' % (artists, s['SNG_TITLE']), - 'uploader': s['ART_NAME'], - 'uploader_id': s['ART_ID'], - 'age_limit': 16 if s.get('EXPLICIT_LYRICS') == '1' else 0, - 'formats': formats, - }) - - return { - '_type': 'playlist', - 'id': playlist_id, - 'title': playlist_title, - 'uploader': playlist_uploader, - 'thumbnail': playlist_thumbnail, - 'entries': entries, - } diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py deleted file mode 100644 index 848d387d1..000000000 --- a/youtube_dl/extractor/dreisat.py +++ /dev/null @@ -1,193 +0,0 @@ -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..utils import ( - int_or_none, - unified_strdate, - xpath_text, - determine_ext, - float_or_none, - ExtractorError, -) - - -class DreiSatIE(InfoExtractor): - IE_NAME = '3sat' - _GEO_COUNTRIES = ['DE'] - _VALID_URL = r'https?://(?:www\.)?3sat\.de/mediathek/(?:(?:index|mediathek)\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)' - _TESTS = [ - { - 'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918', - 'md5': 'be37228896d30a88f315b638900a026e', - 'info_dict': { - 'id': '45918', - 'ext': 'mp4', - 'title': 'Waidmannsheil', - 'description': 'md5:cce00ca1d70e21425e72c86a98a56817', - 'uploader': 'SCHWEIZWEIT', - 'uploader_id': '100000210', - 'upload_date': '20140913' - }, - 'params': { - 'skip_download': True, # m3u8 downloads - } - }, - { - 'url': 'http://www.3sat.de/mediathek/mediathek.php?mode=play&obj=51066', - 'only_matching': True, - }, - ] - - def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None): - param_groups = {} - for param_group in smil.findall(self._xpath_ns('./head/paramGroup', namespace)): - group_id = param_group.get(self._xpath_ns( - 'id', 'http://www.w3.org/XML/1998/namespace')) - params = {} - for param in param_group: - params[param.get('name')] = param.get('value') - param_groups[group_id] = params - - formats = [] - for video in smil.findall(self._xpath_ns('.//video', namespace)): - src = video.get('src') - if not src: - continue - bitrate = int_or_none(self._search_regex(r'_(\d+)k', src, 'bitrate', None)) or float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000) - group_id = video.get('paramGroup') - param_group = param_groups[group_id] - for proto in param_group['protocols'].split(','): - formats.append({ - 'url': '%s://%s' % (proto, param_group['host']), - 'app': param_group['app'], - 'play_path': src, - 'ext': 'flv', - 'format_id': '%s-%d' % (proto, bitrate), - 'tbr': bitrate, - }) - self._sort_formats(formats) - return formats - - def extract_from_xml_url(self, video_id, xml_url): - doc = self._download_xml( - xml_url, video_id, - note='Downloading video info', - errnote='Failed to download video info') - - status_code = xpath_text(doc, './status/statuscode') - if status_code and status_code != 'ok': - if status_code == 'notVisibleAnymore': - message = 'Video %s is not available' % video_id - else: - message = '%s returned error: %s' % (self.IE_NAME, status_code) - raise ExtractorError(message, expected=True) - - title = xpath_text(doc, './/information/title', 'title', True) - - urls = [] - formats = [] - for fnode in doc.findall('.//formitaeten/formitaet'): - video_url = xpath_text(fnode, 'url') - if not video_url or video_url in urls: - continue - urls.append(video_url) - - is_available = 'http://www.metafilegenerator' not in video_url - geoloced = 'static_geoloced_online' in video_url - if not is_available or geoloced: - continue - - format_id = fnode.attrib['basetype'] - format_m = re.match(r'''(?x) - (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_ - (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+) - ''', format_id) - - ext = determine_ext(video_url, None) or format_m.group('container') - - if ext == 'meta': - continue - elif ext == 'smil': - formats.extend(self._extract_smil_formats( - video_url, video_id, fatal=False)) - elif ext == 'm3u8': - # the certificates are misconfigured (see - # https://github.com/ytdl-org/youtube-dl/issues/8665) - if video_url.startswith('https://'): - continue - formats.extend(self._extract_m3u8_formats( - video_url, video_id, 'mp4', 'm3u8_native', - m3u8_id=format_id, fatal=False)) - elif ext == 'f4m': - formats.extend(self._extract_f4m_formats( - video_url, video_id, f4m_id=format_id, fatal=False)) - else: - quality = xpath_text(fnode, './quality') - if quality: - format_id += '-' + quality - - abr = int_or_none(xpath_text(fnode, './audioBitrate'), 1000) - vbr = int_or_none(xpath_text(fnode, './videoBitrate'), 1000) - - tbr = int_or_none(self._search_regex( - r'_(\d+)k', video_url, 'bitrate', None)) - if tbr and vbr and not abr: - abr = tbr - vbr - - formats.append({ - 'format_id': format_id, - 'url': video_url, - 'ext': ext, - 'acodec': format_m.group('acodec'), - 'vcodec': format_m.group('vcodec'), - 'abr': abr, - 'vbr': vbr, - 'tbr': tbr, - 'width': int_or_none(xpath_text(fnode, './width')), - 'height': int_or_none(xpath_text(fnode, './height')), - 'filesize': int_or_none(xpath_text(fnode, './filesize')), - 'protocol': format_m.group('proto').lower(), - }) - - geolocation = xpath_text(doc, './/details/geolocation') - if not formats and geolocation and geolocation != 'none': - self.raise_geo_restricted(countries=self._GEO_COUNTRIES) - - self._sort_formats(formats) - - thumbnails = [] - for node in doc.findall('.//teaserimages/teaserimage'): - thumbnail_url = node.text - if not thumbnail_url: - continue - thumbnail = { - 'url': thumbnail_url, - } - thumbnail_key = node.get('key') - if thumbnail_key: - m = re.match('^([0-9]+)x([0-9]+)$', thumbnail_key) - if m: - thumbnail['width'] = int(m.group(1)) - thumbnail['height'] = int(m.group(2)) - thumbnails.append(thumbnail) - - upload_date = unified_strdate(xpath_text(doc, './/details/airtime')) - - return { - 'id': video_id, - 'title': title, - 'description': xpath_text(doc, './/information/detail'), - 'duration': int_or_none(xpath_text(doc, './/details/lengthSec')), - 'thumbnails': thumbnails, - 'uploader': xpath_text(doc, './/details/originChannelTitle'), - 'uploader_id': xpath_text(doc, './/details/originChannelId'), - 'upload_date': upload_date, - 'formats': formats, - } - - def _real_extract(self, url): - video_id = self._match_id(url) - details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?id=%s' % video_id - return self.extract_from_xml_url(video_id, details_url) diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py deleted file mode 100644 index e435c28e1..000000000 --- a/youtube_dl/extractor/phoenix.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import unicode_literals - -from .dreisat import DreiSatIE - - -class PhoenixIE(DreiSatIE): - IE_NAME = 'phoenix.de' - _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/ - (?: - phoenix/die_sendungen/(?:[^/]+/)? - )? - (?P<id>[0-9]+)''' - _TESTS = [ - { - 'url': 'http://www.phoenix.de/content/884301', - 'md5': 'ed249f045256150c92e72dbb70eadec6', - 'info_dict': { - 'id': '884301', - 'ext': 'mp4', - 'title': 'Michael Krons mit Hans-Werner Sinn', - 'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr', - 'upload_date': '20141025', - 'uploader': 'Im Dialog', - } - }, - { - 'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/869815', - 'only_matching': True, - }, - { - 'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/diskussionen/928234', - 'only_matching': True, - }, - ] - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - internal_id = self._search_regex( - r'<div class="phx_vod" id="phx_vod_([0-9]+)"', - webpage, 'internal video ID') - - api_url = 'http://www.phoenix.de/php/mediaplayer/data/beitrags_details.php?ak=web&id=%s' % internal_id - return self.extract_from_xml_url(video_id, api_url) diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py deleted file mode 100644 index dbe1aaded..000000000 --- a/youtube_dl/extractor/redbulltv.py +++ /dev/null @@ -1,128 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .common import InfoExtractor -from ..compat import compat_HTTPError -from ..utils import ( - float_or_none, - ExtractorError, -) - - -class RedBullTVIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?redbull(?:\.tv|\.com(?:/[^/]+)?(?:/tv)?)(?:/events/[^/]+)?/(?:videos?|live)/(?P<id>AP-\w+)' - _TESTS = [{ - # film - 'url': 'https://www.redbull.tv/video/AP-1Q6XCDTAN1W11', - 'md5': 'fb0445b98aa4394e504b413d98031d1f', - 'info_dict': { - 'id': 'AP-1Q6XCDTAN1W11', - 'ext': 'mp4', - 'title': 'ABC of... WRC - ABC of... S1E6', - 'description': 'md5:5c7ed8f4015c8492ecf64b6ab31e7d31', - 'duration': 1582.04, - }, - }, { - # episode - 'url': 'https://www.redbull.tv/video/AP-1PMHKJFCW1W11', - 'info_dict': { - 'id': 'AP-1PMHKJFCW1W11', - 'ext': 'mp4', - 'title': 'Grime - Hashtags S2E4', - 'description': 'md5:b5f522b89b72e1e23216e5018810bb25', - 'duration': 904.6, - }, - 'params': { - 'skip_download': True, - }, - }, { - 'url': 'https://www.redbull.com/int-en/tv/video/AP-1UWHCAR9S1W11/rob-meets-sam-gaze?playlist=playlists::3f81040a-2f31-4832-8e2e-545b1d39d173', - 'only_matching': True, - }, { - 'url': 'https://www.redbull.com/us-en/videos/AP-1YM9QCYE52111', - 'only_matching': True, - }, { - 'url': 'https://www.redbull.com/us-en/events/AP-1XV2K61Q51W11/live/AP-1XUJ86FDH1W11', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - - session = self._download_json( - 'https://api.redbull.tv/v3/session', video_id, - note='Downloading access token', query={ - 'category': 'personal_computer', - 'os_family': 'http', - }) - if session.get('code') == 'error': - raise ExtractorError('%s said: %s' % ( - self.IE_NAME, session['message'])) - token = session['token'] - - try: - video = self._download_json( - 'https://api.redbull.tv/v3/products/' + video_id, - video_id, note='Downloading video information', - headers={'Authorization': token} - ) - except ExtractorError as e: - if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404: - error_message = self._parse_json( - e.cause.read().decode(), video_id)['error'] - raise ExtractorError('%s said: %s' % ( - self.IE_NAME, error_message), expected=True) - raise - - title = video['title'].strip() - - formats = self._extract_m3u8_formats( - 'https://dms.redbull.tv/v3/%s/%s/playlist.m3u8' % (video_id, token), - video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls') - self._sort_formats(formats) - - subtitles = {} - for resource in video.get('resources', []): - if resource.startswith('closed_caption_'): - splitted_resource = resource.split('_') - if splitted_resource[2]: - subtitles.setdefault('en', []).append({ - 'url': 'https://resources.redbull.tv/%s/%s' % (video_id, resource), - 'ext': splitted_resource[2], - }) - - subheading = video.get('subheading') - if subheading: - title += ' - %s' % subheading - - return { - 'id': video_id, - 'title': title, - 'description': video.get('long_description') or video.get( - 'short_description'), - 'duration': float_or_none(video.get('duration'), scale=1000), - 'formats': formats, - 'subtitles': subtitles, - } - - -class RedBullTVRrnContentIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?redbull(?:\.tv|\.com(?:/[^/]+)?(?:/tv)?)/(?:video|live)/rrn:content:[^:]+:(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})' - _TESTS = [{ - 'url': 'https://www.redbull.com/int-en/tv/video/rrn:content:live-videos:e3e6feb4-e95f-50b7-962a-c70f8fd13c73/mens-dh-finals-fort-william', - 'only_matching': True, - }, { - 'url': 'https://www.redbull.com/int-en/tv/video/rrn:content:videos:a36a0f36-ff1b-5db8-a69d-ee11a14bf48b/tn-ts-style?playlist=rrn:content:event-profiles:83f05926-5de8-5389-b5e4-9bb312d715e8:extras', - 'only_matching': True, - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - - webpage = self._download_webpage(url, display_id) - - video_url = self._og_search_url(webpage) - - return self.url_result( - video_url, ie=RedBullTVIE.ie_key(), - video_id=RedBullTVIE._match_id(video_url)) diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py deleted file mode 100644 index 66088b9ab..000000000 --- a/youtube_dl/extractor/tiktok.py +++ /dev/null @@ -1,138 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .common import InfoExtractor -from ..utils import ( - compat_str, - ExtractorError, - int_or_none, - str_or_none, - try_get, - url_or_none, -) - - -class TikTokBaseIE(InfoExtractor): - def _extract_aweme(self, data): - video = data['video'] - description = str_or_none(try_get(data, lambda x: x['desc'])) - width = int_or_none(try_get(data, lambda x: video['width'])) - height = int_or_none(try_get(data, lambda x: video['height'])) - - format_urls = set() - formats = [] - for format_id in ( - 'play_addr_lowbr', 'play_addr', 'play_addr_h264', - 'download_addr'): - for format in try_get( - video, lambda x: x[format_id]['url_list'], list) or []: - format_url = url_or_none(format) - if not format_url: - continue - if format_url in format_urls: - continue - format_urls.add(format_url) - formats.append({ - 'url': format_url, - 'ext': 'mp4', - 'height': height, - 'width': width, - }) - self._sort_formats(formats) - - thumbnail = url_or_none(try_get( - video, lambda x: x['cover']['url_list'][0], compat_str)) - uploader = try_get(data, lambda x: x['author']['nickname'], compat_str) - timestamp = int_or_none(data.get('create_time')) - comment_count = int_or_none(data.get('comment_count')) or int_or_none( - try_get(data, lambda x: x['statistics']['comment_count'])) - repost_count = int_or_none(try_get( - data, lambda x: x['statistics']['share_count'])) - - aweme_id = data['aweme_id'] - - return { - 'id': aweme_id, - 'title': uploader or aweme_id, - 'description': description, - 'thumbnail': thumbnail, - 'uploader': uploader, - 'timestamp': timestamp, - 'comment_count': comment_count, - 'repost_count': repost_count, - 'formats': formats, - } - - -class TikTokIE(TikTokBaseIE): - _VALID_URL = r'''(?x) - https?:// - (?: - (?:m\.)?tiktok\.com/v| - (?:www\.)?tiktok\.com/share/video - ) - /(?P<id>\d+) - ''' - _TESTS = [{ - 'url': 'https://m.tiktok.com/v/6606727368545406213.html', - 'md5': 'd584b572e92fcd48888051f238022420', - 'info_dict': { - 'id': '6606727368545406213', - 'ext': 'mp4', - 'title': 'Zureeal', - 'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay', - 'thumbnail': r're:^https?://.*~noop.image', - 'uploader': 'Zureeal', - 'timestamp': 1538248586, - 'upload_date': '20180929', - 'comment_count': int, - 'repost_count': int, - } - }, { - 'url': 'https://www.tiktok.com/share/video/6606727368545406213', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage( - 'https://m.tiktok.com/v/%s.html' % video_id, video_id) - data = self._parse_json(self._search_regex( - r'\bdata\s*=\s*({.+?})\s*;', webpage, 'data'), video_id) - return self._extract_aweme(data) - - -class TikTokUserIE(TikTokBaseIE): - _VALID_URL = r'''(?x) - https?:// - (?: - (?:m\.)?tiktok\.com/h5/share/usr| - (?:www\.)?tiktok\.com/share/user - ) - /(?P<id>\d+) - ''' - _TESTS = [{ - 'url': 'https://m.tiktok.com/h5/share/usr/188294915489964032.html', - 'info_dict': { - 'id': '188294915489964032', - }, - 'playlist_mincount': 24, - }, { - 'url': 'https://www.tiktok.com/share/user/188294915489964032', - 'only_matching': True, - }] - - def _real_extract(self, url): - user_id = self._match_id(url) - data = self._download_json( - 'https://m.tiktok.com/h5/share/usr/list/%s/' % user_id, user_id, - query={'_signature': '_'}) - entries = [] - for aweme in data['aweme_list']: - try: - entry = self._extract_aweme(aweme) - except ExtractorError: - continue - entry['extractor_key'] = TikTokIE.ie_key() - entries.append(entry) - return self.playlist_result(entries, user_id) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index 19370f62b..fc351db0d 100755..100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -105,6 +105,7 @@ from .postprocessor import ( FFmpegFixupStretchedPP, FFmpegMergerPP, FFmpegPostProcessor, + FFmpegSubtitlesConvertorPP, get_postprocessor, ) from .version import __version__ @@ -163,7 +164,8 @@ class YoutubeDL(object): simulate: Do not download the video files. format: Video format code. See options.py for more information. outtmpl: Template for output names. - restrictfilenames: Do not allow "&" and spaces in file names + restrictfilenames: Do not allow "&" and spaces in file names. + trim_file_name: Limit length of filename (extension excluded). ignoreerrors: Do not stop on download errors. force_generic_extractor: Force downloader to use the generic extractor nooverwrites: Prevent overwriting files. @@ -228,7 +230,7 @@ class YoutubeDL(object): playlist items. postprocessors: A list of dictionaries, each with an entry * key: The name of the postprocessor. See - youtube_dl/postprocessor/__init__.py for a list. + youtube_dlc/postprocessor/__init__.py for a list. as well as any further keyword arguments for the postprocessor. progress_hooks: A list of functions that get called on download @@ -264,7 +266,7 @@ class YoutubeDL(object): about it, warn otherwise (default) source_address: Client-side IP address to bind to. call_home: Boolean, true iff we are allowed to contact the - youtube-dl servers for debugging. + youtube-dlc servers for debugging. sleep_interval: Number of seconds to sleep before each download when used alone or a lower bound of a range for randomized sleep before each download (minimum possible number @@ -301,7 +303,7 @@ class YoutubeDL(object): use downloader suggested by extractor if None. The following parameters are not used by YoutubeDL itself, they are used by - the downloader (see youtube_dl/downloader/common.py): + the downloader (see youtube_dlc/downloader/common.py): nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle, xattr_set_filesize, external_downloader_args, hls_use_mpegts, @@ -358,6 +360,22 @@ class YoutubeDL(object): } self.params.update(params) self.cache = Cache(self) + self.archive = set() + + """Preload the archive, if any is specified""" + def preload_download_archive(self): + fn = self.params.get('download_archive') + if fn is None: + return False + try: + with locked_file(fn, 'r', encoding='utf-8') as archive_file: + for line in archive_file: + self.archive.add(line.strip()) + except IOError as ioe: + if ioe.errno != errno.ENOENT: + raise + return False + return True def check_deprecated(param, option, suggestion): if self.params.get(param) is not None: @@ -366,6 +384,11 @@ class YoutubeDL(object): return True return False + if self.params.get('verbose'): + self.to_stdout('[debug] Loading archive file %r' % self.params.get('download_archive')) + + preload_download_archive(self) + if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'): if self.params.get('geo_verification_proxy') is None: self.params['geo_verification_proxy'] = self.params['cn_verification_proxy'] @@ -441,7 +464,7 @@ class YoutubeDL(object): if re.match(r'^-[0-9A-Za-z_-]{10}$', a)] if idxs: correct_argv = ( - ['youtube-dl'] + ['youtube-dlc'] + [a for i, a in enumerate(argv) if i not in idxs] + ['--'] + [argv[i] for i in idxs] ) @@ -710,6 +733,16 @@ class YoutubeDL(object): # title "Hello $PATH", we don't want `$PATH` to be expanded. filename = expand_path(outtmpl).replace(sep, '') % template_dict + # https://github.com/blackjack4494/youtube-dlc/issues/85 + trim_file_name = self.params.get('trim_file_name', False) + if trim_file_name: + fn_groups = filename.rsplit('.') + ext = fn_groups[-1] + sub_ext = '' + if len(fn_groups) > 2: + sub_ext = fn_groups[-2] + filename = '.'.join(filter(None, [fn_groups[0][:trim_file_name], sub_ext, ext])) + # Temporary fix for #4787 # 'Treat' all problem characters by passing filename through preferredencoding # to workaround encoding issues with subprocess on python2 @ Windows @@ -721,7 +754,7 @@ class YoutubeDL(object): return None def _match_entry(self, info_dict, incomplete): - """ Returns None iff the file should be downloaded """ + """ Returns None if the file should be downloaded """ video_title = info_dict.get('title', info_dict.get('id', 'video')) if 'title' in info_dict: @@ -1216,11 +1249,13 @@ class YoutubeDL(object): group = _parse_format_selection(tokens, inside_group=True) current_selector = FormatSelector(GROUP, group, []) elif string == '+': - video_selector = current_selector - audio_selector = _parse_format_selection(tokens, inside_merge=True) - if not video_selector or not audio_selector: - raise syntax_error('"+" must be between two format selectors', start) - current_selector = FormatSelector(MERGE, (video_selector, audio_selector), []) + if not current_selector: + raise syntax_error('Unexpected "+"', start) + selector_1 = current_selector + selector_2 = _parse_format_selection(tokens, inside_merge=True) + if not selector_2: + raise syntax_error('Expected a selector', start) + current_selector = FormatSelector(MERGE, (selector_1, selector_2), []) else: raise syntax_error('Operator not recognized: "{0}"'.format(string), start) elif type == tokenize.ENDMARKER: @@ -1305,47 +1340,59 @@ class YoutubeDL(object): if matches: yield matches[-1] elif selector.type == MERGE: - def _merge(formats_info): - format_1, format_2 = [f['format_id'] for f in formats_info] - # The first format must contain the video and the - # second the audio - if formats_info[0].get('vcodec') == 'none': - self.report_error('The first format must ' - 'contain the video, try using ' - '"-f %s+%s"' % (format_2, format_1)) - return - # Formats must be opposite (video+audio) - if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none': - self.report_error( - 'Both formats %s and %s are video-only, you must specify "-f video+audio"' - % (format_1, format_2)) - return - output_ext = ( - formats_info[0]['ext'] - if self.params.get('merge_output_format') is None - else self.params['merge_output_format']) - return { + def _merge(formats_pair): + format_1, format_2 = formats_pair + + formats_info = [] + formats_info.extend(format_1.get('requested_formats', (format_1,))) + formats_info.extend(format_2.get('requested_formats', (format_2,))) + + video_fmts = [fmt_info for fmt_info in formats_info if fmt_info.get('vcodec') != 'none'] + audio_fmts = [fmt_info for fmt_info in formats_info if fmt_info.get('acodec') != 'none'] + + the_only_video = video_fmts[0] if len(video_fmts) == 1 else None + the_only_audio = audio_fmts[0] if len(audio_fmts) == 1 else None + + output_ext = self.params.get('merge_output_format') + if not output_ext: + if the_only_video: + output_ext = the_only_video['ext'] + elif the_only_audio and not video_fmts: + output_ext = the_only_audio['ext'] + else: + output_ext = 'mkv' + + new_dict = { 'requested_formats': formats_info, - 'format': '%s+%s' % (formats_info[0].get('format'), - formats_info[1].get('format')), - 'format_id': '%s+%s' % (formats_info[0].get('format_id'), - formats_info[1].get('format_id')), - 'width': formats_info[0].get('width'), - 'height': formats_info[0].get('height'), - 'resolution': formats_info[0].get('resolution'), - 'fps': formats_info[0].get('fps'), - 'vcodec': formats_info[0].get('vcodec'), - 'vbr': formats_info[0].get('vbr'), - 'stretched_ratio': formats_info[0].get('stretched_ratio'), - 'acodec': formats_info[1].get('acodec'), - 'abr': formats_info[1].get('abr'), + 'format': '+'.join(fmt_info.get('format') for fmt_info in formats_info), + 'format_id': '+'.join(fmt_info.get('format_id') for fmt_info in formats_info), 'ext': output_ext, } - video_selector, audio_selector = map(_build_selector_function, selector.selector) + + if the_only_video: + new_dict.update({ + 'width': the_only_video.get('width'), + 'height': the_only_video.get('height'), + 'resolution': the_only_video.get('resolution'), + 'fps': the_only_video.get('fps'), + 'vcodec': the_only_video.get('vcodec'), + 'vbr': the_only_video.get('vbr'), + 'stretched_ratio': the_only_video.get('stretched_ratio'), + }) + + if the_only_audio: + new_dict.update({ + 'acodec': the_only_audio.get('acodec'), + 'abr': the_only_audio.get('abr'), + }) + + return new_dict + + selector_1, selector_2 = map(_build_selector_function, selector.selector) def selector_function(ctx): for pair in itertools.product( - video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))): + selector_1(copy.deepcopy(ctx)), selector_2(copy.deepcopy(ctx))): yield _merge(pair) filters = [self._build_format_filter(f) for f in selector.filters] @@ -1805,6 +1852,14 @@ class YoutubeDL(object): self.report_error('Cannot write annotations file: ' + annofn) return + def dl(name, info): + fd = get_suitable_downloader(info, self.params)(self, self.params) + for ph in self._progress_hooks: + fd.add_progress_hook(ph) + if self.params.get('verbose'): + self.to_stdout('[debug] Invoking downloader on %r' % info.get('url')) + return fd.download(name, info) + subtitles_are_requested = any([self.params.get('writesubtitles', False), self.params.get('writeautomaticsub')]) @@ -1831,15 +1886,41 @@ class YoutubeDL(object): return else: try: - sub_data = ie._request_webpage( - sub_info['url'], info_dict['id'], note=False).read() - with io.open(encodeFilename(sub_filename), 'wb') as subfile: - subfile.write(sub_data) - except (ExtractorError, IOError, OSError, ValueError) as err: + if self.params.get('sleep_interval_subtitles', False): + dl(sub_filename, sub_info) + else: + sub_data = ie._request_webpage( + sub_info['url'], info_dict['id'], note=False).read() + with io.open(encodeFilename(sub_filename), 'wb') as subfile: + subfile.write(sub_data) + except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning('Unable to download subtitle for "%s": %s' % (sub_lang, error_to_compat_str(err))) continue + if self.params.get('skip_download', False): + if self.params.get('convertsubtitles', False): + subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles')) + filename_real_ext = os.path.splitext(filename)[1][1:] + filename_wo_ext = ( + os.path.splitext(filename)[0] + if filename_real_ext == info_dict['ext'] + else filename) + afilename = '%s.%s' % (filename_wo_ext, self.params.get('convertsubtitles')) + if subconv.available: + info_dict.setdefault('__postprocessors', []) + # info_dict['__postprocessors'].append(subconv) + if os.path.exists(encodeFilename(afilename)): + self.to_screen( + '[download] %s has already been downloaded and ' + 'converted' % afilename) + else: + try: + self.post_process(filename, info_dict) + except (PostProcessingError) as err: + self.report_error('postprocessing: %s' % str(err)) + return + if self.params.get('writeinfojson', False): infofn = replace_extension(filename, 'info.json', info_dict.get('ext')) if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): @@ -1856,14 +1937,6 @@ class YoutubeDL(object): if not self.params.get('skip_download', False): try: - def dl(name, info): - fd = get_suitable_downloader(info, self.params)(self, self.params) - for ph in self._progress_hooks: - fd.add_progress_hook(ph) - if self.params.get('verbose'): - self.to_stdout('[debug] Invoking downloader on %r' % info.get('url')) - return fd.download(name, info) - if info_dict.get('requested_formats') is not None: downloaded = [] success = True @@ -1877,17 +1950,21 @@ class YoutubeDL(object): postprocessors = [merger] def compatible_formats(formats): - video, audio = formats + # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them. + video_formats = [format for format in formats if format.get('vcodec') != 'none'] + audio_formats = [format for format in formats if format.get('acodec') != 'none'] + if len(video_formats) > 2 or len(audio_formats) > 2: + return False + # Check extension - video_ext, audio_ext = video.get('ext'), audio.get('ext') - if video_ext and audio_ext: - COMPATIBLE_EXTS = ( - ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'), - ('webm') - ) - for exts in COMPATIBLE_EXTS: - if video_ext in exts and audio_ext in exts: - return True + exts = set(format.get('ext') for format in formats) + COMPATIBLE_EXTS = ( + set(('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma')), + set(('webm',)), + ) + for ext_sets in COMPATIBLE_EXTS: + if ext_sets.issuperset(exts): + return True # TODO: Check acodec/vcodec return False @@ -2066,7 +2143,7 @@ class YoutubeDL(object): except PostProcessingError as e: self.report_error(e.msg) if files_to_delete and not self.params.get('keepvideo', False): - for old_filename in files_to_delete: + for old_filename in set(files_to_delete): self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename) try: os.remove(encodeFilename(old_filename)) @@ -2102,15 +2179,7 @@ class YoutubeDL(object): if not vid_id: return False # Incomplete video information - try: - with locked_file(fn, 'r', encoding='utf-8') as archive_file: - for line in archive_file: - if line.strip() == vid_id: - return True - except IOError as ioe: - if ioe.errno != errno.ENOENT: - raise - return False + return vid_id in self.archive def record_download_archive(self, info_dict): fn = self.params.get('download_archive') @@ -2120,6 +2189,7 @@ class YoutubeDL(object): assert vid_id with locked_file(fn, 'a', encoding='utf-8') as archive_file: archive_file.write(vid_id + '\n') + self.archive.add(vid_id) @staticmethod def format_resolution(format, default='unknown'): @@ -2256,7 +2326,7 @@ class YoutubeDL(object): self.get_encoding())) write_string(encoding_str, encoding=None) - self._write_string('[debug] youtube-dl version ' + __version__ + '\n') + self._write_string('[debug] youtube-dlc version ' + __version__ + '\n') if _LAZY_LOADER: self._write_string('[debug] Lazy loading extractors enabled' + '\n') try: @@ -2354,7 +2424,7 @@ class YoutubeDL(object): file_handler = compat_urllib_request.FileHandler() def file_open(*args, **kwargs): - raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons') + raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dlc for security reasons') file_handler.file_open = file_open opener = compat_urllib_request.build_opener( diff --git a/youtube_dl/__init__.py b/youtube_dlc/__init__.py index 9a659fc65..105786bc0 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dlc/__init__.py @@ -53,7 +53,7 @@ def _real_main(argv=None): workaround_optparse_bug9161() - setproctitle('youtube-dl') + setproctitle('youtube-dlc') parser, opts, args = parseOpts(argv) @@ -209,6 +209,9 @@ def _real_main(argv=None): opts.audioquality = opts.audioquality.strip('k').strip('K') if not opts.audioquality.isdigit(): parser.error('invalid audio quality specified') + if opts.remuxvideo is not None: + if opts.remuxvideo not in ['mp4', 'mkv']: + parser.error('invalid video container format specified') if opts.recodevideo is not None: if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']: parser.error('invalid video recode format specified') @@ -261,6 +264,11 @@ def _real_main(argv=None): 'preferredquality': opts.audioquality, 'nopostoverwrites': opts.nopostoverwrites, }) + if opts.remuxvideo: + postprocessors.append({ + 'key': 'FFmpegVideoRemuxer', + 'preferedformat': opts.remuxvideo, + }) if opts.recodevideo: postprocessors.append({ 'key': 'FFmpegVideoConvertor', @@ -315,6 +323,7 @@ def _real_main(argv=None): else match_filter_func(opts.match_filter)) ydl_opts = { + 'convertsubtitles': opts.convertsubtitles, 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, @@ -381,6 +390,7 @@ def _real_main(argv=None): 'rejecttitle': decodeOption(opts.rejecttitle), 'max_downloads': opts.max_downloads, 'prefer_free_formats': opts.prefer_free_formats, + 'trim_file_name': opts.trim_file_name, 'verbose': opts.verbose, 'dump_intermediate_pages': opts.dump_intermediate_pages, 'write_pages': opts.write_pages, @@ -405,7 +415,9 @@ def _real_main(argv=None): 'prefer_ffmpeg': opts.prefer_ffmpeg, 'include_ads': opts.include_ads, 'default_search': opts.default_search, + 'dynamic_mpd': opts.dynamic_mpd, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, + 'youtube_include_hls_manifest': opts.youtube_include_hls_manifest, 'encoding': opts.encoding, 'extract_flat': opts.extract_flat, 'mark_watched': opts.mark_watched, @@ -416,6 +428,7 @@ def _real_main(argv=None): 'call_home': opts.call_home, 'sleep_interval': opts.sleep_interval, 'max_sleep_interval': opts.max_sleep_interval, + 'sleep_interval_subtitles': opts.sleep_interval_subtitles, 'external_downloader': opts.external_downloader, 'list_thumbnails': opts.list_thumbnails, 'playlist_items': opts.playlist_items, @@ -455,7 +468,7 @@ def _real_main(argv=None): ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv) parser.error( 'You must provide at least one URL.\n' - 'Type youtube-dl --help to see a list of all options.') + 'Type youtube-dlc --help to see a list of all options.') try: if opts.load_info_filename is not None: diff --git a/youtube_dl/__main__.py b/youtube_dlc/__main__.py index 138f5fbec..0e7601686 100755..100644 --- a/youtube_dl/__main__.py +++ b/youtube_dlc/__main__.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals # Execute with -# $ python youtube_dl/__main__.py (2.6+) -# $ python -m youtube_dl (2.7+) +# $ python youtube_dlc/__main__.py (2.6+) +# $ python -m youtube_dlc (2.7+) import sys @@ -13,7 +13,7 @@ if __package__ is None and not hasattr(sys, 'frozen'): path = os.path.realpath(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(os.path.dirname(path))) -import youtube_dl +import youtube_dlc if __name__ == '__main__': - youtube_dl.main() + youtube_dlc.main() diff --git a/youtube_dl/aes.py b/youtube_dlc/aes.py index 461bb6d41..461bb6d41 100644 --- a/youtube_dl/aes.py +++ b/youtube_dlc/aes.py diff --git a/youtube_dl/cache.py b/youtube_dlc/cache.py index 7bdade1bd..ada6aa1f2 100644 --- a/youtube_dl/cache.py +++ b/youtube_dlc/cache.py @@ -23,7 +23,7 @@ class Cache(object): res = self._ydl.params.get('cachedir') if res is None: cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache') - res = os.path.join(cache_root, 'youtube-dl') + res = os.path.join(cache_root, 'youtube-dlc') return expand_path(res) def _get_cache_fn(self, section, key, dtype): diff --git a/youtube_dl/compat.py b/youtube_dlc/compat.py index 0ee9bc760..1cf7efed6 100644 --- a/youtube_dl/compat.py +++ b/youtube_dlc/compat.py @@ -2973,7 +2973,7 @@ else: if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0): # PyPy2 prior to version 5.4.0 expects byte strings as Windows function - # names, see the original PyPy issue [1] and the youtube-dl one [2]. + # names, see the original PyPy issue [1] and the youtube-dlc one [2]. # 1. https://bitbucket.org/pypy/pypy/issues/2360/windows-ctypescdll-typeerror-function-name # 2. https://github.com/ytdl-org/youtube-dl/pull/4392 def compat_ctypes_WINFUNCTYPE(*args, **kwargs): diff --git a/youtube_dl/downloader/__init__.py b/youtube_dlc/downloader/__init__.py index 2e485df9d..4ae81f516 100644 --- a/youtube_dl/downloader/__init__.py +++ b/youtube_dlc/downloader/__init__.py @@ -8,6 +8,7 @@ from .rtmp import RtmpFD from .dash import DashSegmentsFD from .rtsp import RtspFD from .ism import IsmFD +from .youtube_live_chat import YoutubeLiveChatReplayFD from .external import ( get_external_downloader, FFmpegFD, @@ -26,6 +27,7 @@ PROTOCOL_MAP = { 'f4m': F4mFD, 'http_dash_segments': DashSegmentsFD, 'ism': IsmFD, + 'youtube_live_chat_replay': YoutubeLiveChatReplayFD, } diff --git a/youtube_dl/downloader/common.py b/youtube_dlc/downloader/common.py index 1cdba89cd..31c286458 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dlc/downloader/common.py @@ -243,7 +243,7 @@ class FileDownloader(object): else: clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r') self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) - self.to_console_title('youtube-dl ' + msg) + self.to_console_title('youtube-dlc ' + msg) def report_progress(self, s): if s['status'] == 'finished': diff --git a/youtube_dl/downloader/dash.py b/youtube_dlc/downloader/dash.py index c6d674bc6..c6d674bc6 100644 --- a/youtube_dl/downloader/dash.py +++ b/youtube_dlc/downloader/dash.py diff --git a/youtube_dl/downloader/external.py b/youtube_dlc/downloader/external.py index c31f8910a..c31f8910a 100644 --- a/youtube_dl/downloader/external.py +++ b/youtube_dlc/downloader/external.py diff --git a/youtube_dl/downloader/f4m.py b/youtube_dlc/downloader/f4m.py index 8dd3c2eeb..8dd3c2eeb 100644 --- a/youtube_dl/downloader/f4m.py +++ b/youtube_dlc/downloader/f4m.py diff --git a/youtube_dl/downloader/fragment.py b/youtube_dlc/downloader/fragment.py index 02f35459e..9339b3a62 100644 --- a/youtube_dl/downloader/fragment.py +++ b/youtube_dlc/downloader/fragment.py @@ -32,9 +32,9 @@ class FragmentFD(FileDownloader): keep_fragments: Keep downloaded fragments on disk after downloading is finished - For each incomplete fragment download youtube-dl keeps on disk a special + For each incomplete fragment download youtube-dlc keeps on disk a special bookkeeping file with download state and metadata (in future such files will - be used for any incomplete download handled by youtube-dl). This file is + be used for any incomplete download handled by youtube-dlc). This file is used to properly handle resuming, check download file consistency and detect potential errors. The file has a .ytdl extension and represents a standard JSON file of the following format: diff --git a/youtube_dl/downloader/hls.py b/youtube_dlc/downloader/hls.py index 84bc34928..0f2c06f40 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dlc/downloader/hls.py @@ -141,7 +141,7 @@ class HlsFD(FragmentFD): count = 0 headers = info_dict.get('http_headers', {}) if byte_range: - headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end']) + headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1) while count <= fragment_retries: try: success, frag_content = self._download_fragment( diff --git a/youtube_dl/downloader/http.py b/youtube_dlc/downloader/http.py index 5046878df..96379caf1 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dlc/downloader/http.py @@ -106,7 +106,12 @@ class HttpFD(FileDownloader): set_range(request, range_start, range_end) # Establish connection try: - ctx.data = self.ydl.urlopen(request) + try: + ctx.data = self.ydl.urlopen(request) + except (compat_urllib_error.URLError, ) as err: + if isinstance(err.reason, socket.timeout): + raise RetryDownload(err) + raise err # When trying to resume, Content-Range HTTP header of response has to be checked # to match the value of requested Range HTTP header. This is due to a webservers # that don't support resuming and serve a whole file with no Content-Range @@ -218,9 +223,10 @@ class HttpFD(FileDownloader): def retry(e): to_stdout = ctx.tmpfilename == '-' - if not to_stdout: - ctx.stream.close() - ctx.stream = None + if ctx.stream is not None: + if not to_stdout: + ctx.stream.close() + ctx.stream = None ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename)) raise RetryDownload(e) @@ -233,9 +239,11 @@ class HttpFD(FileDownloader): except socket.timeout as e: retry(e) except socket.error as e: - if e.errno not in (errno.ECONNRESET, errno.ETIMEDOUT): - raise - retry(e) + # SSLError on python 2 (inherits socket.error) may have + # no errno set but this error message + if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or getattr(e, 'message', None) == 'The read operation timed out': + retry(e) + raise byte_counter += len(data_block) diff --git a/youtube_dl/downloader/ism.py b/youtube_dlc/downloader/ism.py index 1ca666b4a..1ca666b4a 100644 --- a/youtube_dl/downloader/ism.py +++ b/youtube_dlc/downloader/ism.py diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dlc/downloader/rtmp.py index fbb7f51b0..fbb7f51b0 100644 --- a/youtube_dl/downloader/rtmp.py +++ b/youtube_dlc/downloader/rtmp.py diff --git a/youtube_dl/downloader/rtsp.py b/youtube_dlc/downloader/rtsp.py index 939358b2a..939358b2a 100644 --- a/youtube_dl/downloader/rtsp.py +++ b/youtube_dlc/downloader/rtsp.py diff --git a/youtube_dlc/downloader/youtube_live_chat.py b/youtube_dlc/downloader/youtube_live_chat.py new file mode 100644 index 000000000..4932dd9c5 --- /dev/null +++ b/youtube_dlc/downloader/youtube_live_chat.py @@ -0,0 +1,94 @@ +from __future__ import division, unicode_literals + +import re +import json + +from .fragment import FragmentFD + + +class YoutubeLiveChatReplayFD(FragmentFD): + """ Downloads YouTube live chat replays fragment by fragment """ + + FD_NAME = 'youtube_live_chat_replay' + + def real_download(self, filename, info_dict): + video_id = info_dict['video_id'] + self.to_screen('[%s] Downloading live chat' % self.FD_NAME) + + test = self.params.get('test', False) + + ctx = { + 'filename': filename, + 'live': True, + 'total_frags': None, + } + + def dl_fragment(url): + headers = info_dict.get('http_headers', {}) + return self._download_fragment(ctx, url, info_dict, headers) + + def parse_yt_initial_data(data): + window_patt = b'window\\["ytInitialData"\\]\\s*=\\s*(.*?)(?<=});' + var_patt = b'var\\s+ytInitialData\\s*=\\s*(.*?)(?<=});' + for patt in window_patt, var_patt: + try: + raw_json = re.search(patt, data).group(1) + return json.loads(raw_json) + except AttributeError: + continue + + self._prepare_and_start_frag_download(ctx) + + success, raw_fragment = dl_fragment( + 'https://www.youtube.com/watch?v={}'.format(video_id)) + if not success: + return False + data = parse_yt_initial_data(raw_fragment) + continuation_id = data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'] + # no data yet but required to call _append_fragment + self._append_fragment(ctx, b'') + + first = True + offset = None + while continuation_id is not None: + data = None + if first: + url = 'https://www.youtube.com/live_chat_replay?continuation={}'.format(continuation_id) + success, raw_fragment = dl_fragment(url) + if not success: + return False + data = parse_yt_initial_data(raw_fragment) + else: + url = ('https://www.youtube.com/live_chat_replay/get_live_chat_replay' + + '?continuation={}'.format(continuation_id) + + '&playerOffsetMs={}'.format(offset - 5000) + + '&hidden=false' + + '&pbj=1') + success, raw_fragment = dl_fragment(url) + if not success: + return False + data = json.loads(raw_fragment)['response'] + + first = False + continuation_id = None + + live_chat_continuation = data['continuationContents']['liveChatContinuation'] + offset = None + processed_fragment = bytearray() + if 'actions' in live_chat_continuation: + for action in live_chat_continuation['actions']: + if 'replayChatItemAction' in action: + replay_chat_item_action = action['replayChatItemAction'] + offset = int(replay_chat_item_action['videoOffsetTimeMsec']) + processed_fragment.extend( + json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n') + continuation_id = live_chat_continuation['continuations'][0]['liveChatReplayContinuationData']['continuation'] + + self._append_fragment(ctx, processed_fragment) + + if test or offset is None: + break + + self._finish_frag_download(ctx) + + return True diff --git a/youtube_dl/extractor/__init__.py b/youtube_dlc/extractor/__init__.py index 18d8dbcd6..18d8dbcd6 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dlc/extractor/__init__.py diff --git a/youtube_dl/extractor/abc.py b/youtube_dlc/extractor/abc.py index 6637f4f35..3e202168e 100644 --- a/youtube_dl/extractor/abc.py +++ b/youtube_dlc/extractor/abc.py @@ -12,6 +12,7 @@ from ..utils import ( js_to_json, int_or_none, parse_iso8601, + str_or_none, try_get, unescapeHTML, update_url_query, @@ -20,7 +21,7 @@ from ..utils import ( class ABCIE(InfoExtractor): IE_NAME = 'abc.net.au' - _VALID_URL = r'https?://(?:www\.)?abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)' + _VALID_URL = r'https?://(?:www\.)?abc\.net\.au/(?:news|btn)/(?:[^/]+/){1,4}(?P<id>\d{5,})' _TESTS = [{ 'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334', @@ -34,7 +35,7 @@ class ABCIE(InfoExtractor): 'skip': 'this video has expired', }, { 'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326', - 'md5': 'db2a5369238b51f9811ad815b69dc086', + 'md5': '4ebd61bdc82d9a8b722f64f1f4b4d121', 'info_dict': { 'id': 'NvqvPeNZsHU', 'ext': 'mp4', @@ -58,39 +59,102 @@ class ABCIE(InfoExtractor): }, { 'url': 'http://www.abc.net.au/news/2015-10-19/6866214', 'only_matching': True, + }, { + 'url': 'https://www.abc.net.au/btn/classroom/wwi-centenary/10527914', + 'info_dict': { + 'id': '10527914', + 'ext': 'mp4', + 'title': 'WWI Centenary', + 'description': 'md5:c2379ec0ca84072e86b446e536954546', + } + }, { + 'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074', + 'info_dict': { + 'id': '12342074', + 'ext': 'mp4', + 'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia', + 'description': 'md5:2961a17dc53abc558589ccd0fb8edd6f', + } + }, { + 'url': 'https://www.abc.net.au/btn/newsbreak/btn-newsbreak-20200814/12560476', + 'info_dict': { + 'id': 'tDL8Ld4dK_8', + 'ext': 'mp4', + 'title': 'Fortnite Banned From Apple and Google App Stores', + 'description': 'md5:a6df3f36ce8f816b74af4bd6462f5651', + 'upload_date': '20200813', + 'uploader': 'Behind the News', + 'uploader_id': 'behindthenews', + } }] def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - mobj = re.search( - r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);', - webpage) + mobj = re.search(r'<a\s+href="(?P<url>[^"]+)"\s+data-duration="\d+"\s+title="Download audio directly">', webpage) + if mobj: + urls_info = mobj.groupdict() + youtube = False + video = False + else: + mobj = re.search(r'<a href="(?P<url>http://www\.youtube\.com/watch\?v=[^"]+)"><span><strong>External Link:</strong>', + webpage) + if mobj is None: + mobj = re.search(r'<iframe width="100%" src="(?P<url>//www\.youtube-nocookie\.com/embed/[^?"]+)', webpage) + if mobj: + urls_info = mobj.groupdict() + youtube = True + video = True + if mobj is None: - expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?<span>(.+?)</span>', webpage, 'expired', None) - if expired: - raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True) - raise ExtractorError('Unable to extract video urls') + mobj = re.search(r'(?P<type>)"sources": (?P<json_data>\[[^\]]+\]),', webpage) + if mobj is None: + mobj = re.search( + r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);', + webpage) + if mobj is None: + expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?<span>(.+?)</span>', webpage, 'expired', None) + if expired: + raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True) + raise ExtractorError('Unable to extract video urls') - urls_info = self._parse_json( - mobj.group('json_data'), video_id, transform_source=js_to_json) + urls_info = self._parse_json( + mobj.group('json_data'), video_id, transform_source=js_to_json) + youtube = mobj.group('type') == 'YouTube' + video = mobj.group('type') == 'Video' or urls_info[0]['contentType'] == 'video/mp4' if not isinstance(urls_info, list): urls_info = [urls_info] - if mobj.group('type') == 'YouTube': + if youtube: return self.playlist_result([ self.url_result(url_info['url']) for url_info in urls_info]) - formats = [{ - 'url': url_info['url'], - 'vcodec': url_info.get('codec') if mobj.group('type') == 'Video' else 'none', - 'width': int_or_none(url_info.get('width')), - 'height': int_or_none(url_info.get('height')), - 'tbr': int_or_none(url_info.get('bitrate')), - 'filesize': int_or_none(url_info.get('filesize')), - } for url_info in urls_info] + formats = [] + for url_info in urls_info: + height = int_or_none(url_info.get('height')) + bitrate = int_or_none(url_info.get('bitrate')) + width = int_or_none(url_info.get('width')) + format_id = None + mobj = re.search(r'_(?:(?P<height>\d+)|(?P<bitrate>\d+)k)\.mp4$', url_info['url']) + if mobj: + height_from_url = mobj.group('height') + if height_from_url: + height = height or int_or_none(height_from_url) + width = width or int_or_none(url_info.get('label')) + else: + bitrate = bitrate or int_or_none(mobj.group('bitrate')) + format_id = str_or_none(url_info.get('label')) + formats.append({ + 'url': url_info['url'], + 'vcodec': url_info.get('codec') if video else 'none', + 'width': width, + 'height': height, + 'tbr': bitrate, + 'filesize': int_or_none(url_info.get('filesize')), + 'format_id': format_id + }) self._sort_formats(formats) diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dlc/extractor/abcnews.py index 8b407bf9c..8b407bf9c 100644 --- a/youtube_dl/extractor/abcnews.py +++ b/youtube_dlc/extractor/abcnews.py diff --git a/youtube_dl/extractor/abcotvs.py b/youtube_dlc/extractor/abcotvs.py index 0bc69a64f..0bc69a64f 100644 --- a/youtube_dl/extractor/abcotvs.py +++ b/youtube_dlc/extractor/abcotvs.py diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dlc/extractor/academicearth.py index 34095501c..34095501c 100644 --- a/youtube_dl/extractor/academicearth.py +++ b/youtube_dlc/extractor/academicearth.py diff --git a/youtube_dl/extractor/acast.py b/youtube_dlc/extractor/acast.py index b17c792d2..b17c792d2 100644 --- a/youtube_dl/extractor/acast.py +++ b/youtube_dlc/extractor/acast.py diff --git a/youtube_dl/extractor/adn.py b/youtube_dlc/extractor/adn.py index c95ad2173..c95ad2173 100644 --- a/youtube_dl/extractor/adn.py +++ b/youtube_dlc/extractor/adn.py diff --git a/youtube_dl/extractor/adobeconnect.py b/youtube_dlc/extractor/adobeconnect.py index 728549eb9..728549eb9 100644 --- a/youtube_dl/extractor/adobeconnect.py +++ b/youtube_dlc/extractor/adobeconnect.py diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dlc/extractor/adobepass.py index 38dca1b0a..38dca1b0a 100644 --- a/youtube_dl/extractor/adobepass.py +++ b/youtube_dlc/extractor/adobepass.py diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dlc/extractor/adobetv.py index 80060f037..80060f037 100644 --- a/youtube_dl/extractor/adobetv.py +++ b/youtube_dlc/extractor/adobetv.py diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dlc/extractor/adultswim.py index 8d1d9ac7d..8d1d9ac7d 100644 --- a/youtube_dl/extractor/adultswim.py +++ b/youtube_dlc/extractor/adultswim.py diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dlc/extractor/aenetworks.py index 611b948f5..611b948f5 100644 --- a/youtube_dl/extractor/aenetworks.py +++ b/youtube_dlc/extractor/aenetworks.py diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dlc/extractor/afreecatv.py index 6275e5209..6275e5209 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dlc/extractor/afreecatv.py diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dlc/extractor/airmozilla.py index 9e38136b4..9e38136b4 100644 --- a/youtube_dl/extractor/airmozilla.py +++ b/youtube_dlc/extractor/airmozilla.py diff --git a/youtube_dl/extractor/aliexpress.py b/youtube_dlc/extractor/aliexpress.py index 6f241e683..6f241e683 100644 --- a/youtube_dl/extractor/aliexpress.py +++ b/youtube_dlc/extractor/aliexpress.py diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dlc/extractor/aljazeera.py index c68be3134..c68be3134 100644 --- a/youtube_dl/extractor/aljazeera.py +++ b/youtube_dlc/extractor/aljazeera.py diff --git a/youtube_dl/extractor/allocine.py b/youtube_dlc/extractor/allocine.py index cd533acfc..cd533acfc 100644 --- a/youtube_dl/extractor/allocine.py +++ b/youtube_dlc/extractor/allocine.py diff --git a/youtube_dl/extractor/alphaporno.py b/youtube_dlc/extractor/alphaporno.py index 3a6d99f6b..3a6d99f6b 100644 --- a/youtube_dl/extractor/alphaporno.py +++ b/youtube_dlc/extractor/alphaporno.py diff --git a/youtube_dlc/extractor/alura.py b/youtube_dlc/extractor/alura.py new file mode 100644 index 000000000..36b4d95b3 --- /dev/null +++ b/youtube_dlc/extractor/alura.py @@ -0,0 +1,180 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + +from ..compat import ( + compat_urlparse, +) + +from ..utils import ( + urlencode_postdata, + urljoin, + int_or_none, + clean_html, + ExtractorError +) + + +class AluraIE(InfoExtractor): + _VALID_URL = r'https?://(?:cursos\.)?alura\.com\.br/course/(?P<course_name>[^/]+)/task/(?P<id>\d+)' + _LOGIN_URL = 'https://cursos.alura.com.br/loginForm?urlAfterLogin=/loginForm' + _VIDEO_URL = 'https://cursos.alura.com.br/course/%s/task/%s/video' + _NETRC_MACHINE = 'alura' + _TESTS = [{ + 'url': 'https://cursos.alura.com.br/course/clojure-mutabilidade-com-atoms-e-refs/task/60095', + 'info_dict': { + 'id': '60095', + 'ext': 'mp4', + 'title': 'Referências, ref-set e alter' + }, + 'skip': 'Requires alura account credentials'}, + { + # URL without video + 'url': 'https://cursos.alura.com.br/course/clojure-mutabilidade-com-atoms-e-refs/task/60098', + 'only_matching': True}, + { + 'url': 'https://cursos.alura.com.br/course/fundamentos-market-digital/task/55219', + 'only_matching': True} + ] + + def _real_extract(self, url): + + video_id = self._match_id(url) + course = self._search_regex(self._VALID_URL, url, 'post url', group='course_name') + video_url = self._VIDEO_URL % (course, video_id) + + video_dict = self._download_json(video_url, video_id, 'Searching for videos') + + if video_dict: + webpage = self._download_webpage(url, video_id) + video_title = clean_html(self._search_regex( + r'<span[^>]+class=(["\'])task-body-header-title-text\1[^>]*>(?P<title>[^<]+)', + webpage, 'title', group='title')) + + formats = [] + for video_obj in video_dict: + video_url_m3u8 = video_obj.get('link') + video_format = self._extract_m3u8_formats( + video_url_m3u8, None, 'mp4', entry_protocol='m3u8_native', + m3u8_id='hls', fatal=False) + for f in video_format: + m = re.search(r'^[\w \W]*-(?P<res>\w*).mp4[\W \w]*', f['url']) + if m: + if not f.get('height'): + f['height'] = int('720' if m.group('res') == 'hd' else '480') + formats.extend(video_format) + + self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id')) + + return { + 'id': video_id, + 'title': video_title, + "formats": formats + } + + def _real_initialize(self): + self._login() + + def _login(self): + username, password = self._get_login_info() + if username is None: + return + pass + + login_page = self._download_webpage( + self._LOGIN_URL, None, 'Downloading login popup') + + def is_logged(webpage): + return any(re.search(p, webpage) for p in ( + r'href=[\"|\']?/signout[\"|\']', + r'>Logout<')) + + # already logged in + if is_logged(login_page): + return + + login_form = self._hidden_inputs(login_page) + + login_form.update({ + 'username': username, + 'password': password, + }) + + post_url = self._search_regex( + r'<form[^>]+class=["|\']signin-form["|\'] action=["|\'](?P<url>.+?)["|\']', login_page, + 'post url', default=self._LOGIN_URL, group='url') + + if not post_url.startswith('http'): + post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url) + + response = self._download_webpage( + post_url, None, 'Logging in', + data=urlencode_postdata(login_form), + headers={'Content-Type': 'application/x-www-form-urlencoded'}) + + if not is_logged(response): + error = self._html_search_regex( + r'(?s)<p[^>]+class="alert-message[^"]*">(.+?)</p>', + response, 'error message', default=None) + if error: + raise ExtractorError('Unable to login: %s' % error, expected=True) + raise ExtractorError('Unable to log in') + + +class AluraCourseIE(AluraIE): + + _VALID_URL = r'https?://(?:cursos\.)?alura\.com\.br/course/(?P<id>[^/]+)' + _LOGIN_URL = 'https://cursos.alura.com.br/loginForm?urlAfterLogin=/loginForm' + _NETRC_MACHINE = 'aluracourse' + _TESTS = [{ + 'url': 'https://cursos.alura.com.br/course/clojure-mutabilidade-com-atoms-e-refs', + 'only_matching': True, + }] + + @classmethod + def suitable(cls, url): + return False if AluraIE.suitable(url) else super(AluraCourseIE, cls).suitable(url) + + def _real_extract(self, url): + + course_path = self._match_id(url) + webpage = self._download_webpage(url, course_path) + + course_title = self._search_regex( + r'<h1.*?>(.*?)<strong>(?P<course_title>.*?)</strong></h[0-9]>', webpage, + 'course title', default=course_path, group='course_title') + + entries = [] + if webpage: + for path in re.findall(r'<a\b(?=[^>]* class="[^"]*(?<=[" ])courseSectionList-section[" ])(?=[^>]* href="([^"]*))', webpage): + page_url = urljoin(url, path) + section_path = self._download_webpage(page_url, course_path) + for path_video in re.findall(r'<a\b(?=[^>]* class="[^"]*(?<=[" ])task-menu-nav-item-link-VIDEO[" ])(?=[^>]* href="([^"]*))', section_path): + chapter = clean_html( + self._search_regex( + r'<h3[^>]+class=(["\'])task-menu-section-title-text\1[^>]*>(?P<chapter>[^<]+)', + section_path, + 'chapter', + group='chapter')) + + chapter_number = int_or_none( + self._search_regex( + r'<span[^>]+class=(["\'])task-menu-section-title-number[^>]*>(.*?)<strong>(?P<chapter_number>[^<]+)</strong>', + section_path, + 'chapter number', + group='chapter_number')) + video_url = urljoin(url, path_video) + + entry = { + '_type': 'url_transparent', + 'id': self._match_id(video_url), + 'url': video_url, + 'id_key': self.ie_key(), + 'chapter': chapter, + 'chapter_number': chapter_number + } + entries.append(entry) + return self.playlist_result(entries, course_path, course_title) diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dlc/extractor/amcnetworks.py index 6fb3d6c53..6fb3d6c53 100644 --- a/youtube_dl/extractor/amcnetworks.py +++ b/youtube_dlc/extractor/amcnetworks.py diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dlc/extractor/americastestkitchen.py index 9c9d77ae1..9c9d77ae1 100644 --- a/youtube_dl/extractor/americastestkitchen.py +++ b/youtube_dlc/extractor/americastestkitchen.py diff --git a/youtube_dl/extractor/amp.py b/youtube_dlc/extractor/amp.py index 7ff098cfa..7ff098cfa 100644 --- a/youtube_dl/extractor/amp.py +++ b/youtube_dlc/extractor/amp.py diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dlc/extractor/animeondemand.py index 00ce684d1..00ce684d1 100644 --- a/youtube_dl/extractor/animeondemand.py +++ b/youtube_dlc/extractor/animeondemand.py diff --git a/youtube_dl/extractor/anvato.py b/youtube_dlc/extractor/anvato.py index 84e841035..84e841035 100644 --- a/youtube_dl/extractor/anvato.py +++ b/youtube_dlc/extractor/anvato.py diff --git a/youtube_dl/extractor/aol.py b/youtube_dlc/extractor/aol.py index e87994a6a..e87994a6a 100644 --- a/youtube_dl/extractor/aol.py +++ b/youtube_dlc/extractor/aol.py diff --git a/youtube_dl/extractor/apa.py b/youtube_dlc/extractor/apa.py index 98ccdaa4a..98ccdaa4a 100644 --- a/youtube_dl/extractor/apa.py +++ b/youtube_dlc/extractor/apa.py diff --git a/youtube_dl/extractor/aparat.py b/youtube_dlc/extractor/aparat.py index 883dcee7a..883dcee7a 100644 --- a/youtube_dl/extractor/aparat.py +++ b/youtube_dlc/extractor/aparat.py diff --git a/youtube_dl/extractor/appleconnect.py b/youtube_dlc/extractor/appleconnect.py index a84b8b1eb..a84b8b1eb 100644 --- a/youtube_dl/extractor/appleconnect.py +++ b/youtube_dlc/extractor/appleconnect.py diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dlc/extractor/appletrailers.py index a9ef733e0..b5ed2b88b 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dlc/extractor/appletrailers.py @@ -199,7 +199,7 @@ class AppleTrailersIE(InfoExtractor): 'upload_date': upload_date, 'uploader_id': uploader_id, 'http_headers': { - 'User-Agent': 'QuickTime compatible (youtube-dl)', + 'User-Agent': 'QuickTime compatible (youtube-dlc)', }, }) diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dlc/extractor/archiveorg.py index c79c58e82..c79c58e82 100644 --- a/youtube_dl/extractor/archiveorg.py +++ b/youtube_dlc/extractor/archiveorg.py diff --git a/youtube_dl/extractor/ard.py b/youtube_dlc/extractor/ard.py index 5b7b2dd6d..6f1e477a9 100644 --- a/youtube_dl/extractor/ard.py +++ b/youtube_dlc/extractor/ard.py @@ -62,6 +62,45 @@ class ARDMediathekBaseIE(InfoExtractor): 'subtitles': subtitles, } + def _ARD_extract_episode_info(self, title): + """Try to extract season/episode data from the title.""" + res = {} + if not title: + return res + + for pattern in [ + # Pattern for title like "Homo sapiens (S06/E07) - Originalversion" + # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw + r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*', + # E.g.: title="Fritjof aus Norwegen (2) (AD)" + # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/ + r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*', + r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*', + # E.g.: title="Folge 25/42: Symmetrie" + # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/ + # E.g.: title="Folge 1063 - Vertrauen" + # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/ + r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*', + ]: + m = re.match(pattern, title) + if m: + groupdict = m.groupdict() + res['season_number'] = int_or_none(groupdict.get('season_number')) + res['episode_number'] = int_or_none(groupdict.get('episode_number')) + res['episode'] = str_or_none(groupdict.get('episode')) + # Build the episode title by removing numeric episode information: + if groupdict.get('ep_info') and not res['episode']: + res['episode'] = str_or_none( + title.replace(groupdict.get('ep_info'), '')) + if res['episode']: + res['episode'] = res['episode'].strip() + break + + # As a fallback use the whole title as the episode name: + if not res.get('episode'): + res['episode'] = title.strip() + return res + def _extract_formats(self, media_info, video_id): type_ = media_info.get('_type') media_array = media_info.get('_mediaArray', []) @@ -244,6 +283,7 @@ class ARDMediathekIE(ARDMediathekBaseIE): 'description': description, 'thumbnail': thumbnail, }) + info.update(self._ARD_extract_episode_info(info['title'])) return info @@ -313,7 +353,7 @@ class ARDIE(InfoExtractor): class ARDBetaMediathekIE(ARDMediathekBaseIE): - _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?P<client>[^/]+)/(?:player|live|video)/(?P<display_id>(?:[^/]+/)*)(?P<video_id>[a-zA-Z0-9]+)' + _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?P<client>[^/]+)/(?P<mode>player|live|video|sendung|sammlung)/(?P<display_id>(?:[^/]+/)*)(?P<video_id>[a-zA-Z0-9]+)' _TESTS = [{ 'url': 'https://ardmediathek.de/ard/video/die-robuste-roswita/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE', 'md5': 'dfdc87d2e7e09d073d5a80770a9ce88f', @@ -343,8 +383,112 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE): }, { 'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg', 'only_matching': True, + }, { + # playlist of type 'sendung' + 'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/', + 'only_matching': True, + }, { + # playlist of type 'sammlung' + 'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/', + 'only_matching': True, }] + def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, pageNumber): + """ Query the ARD server for playlist information + and returns the data in "raw" format """ + if mode == 'sendung': + graphQL = json.dumps({ + 'query': '''{ + showPage( + client: "%s" + showId: "%s" + pageNumber: %d + ) { + pagination { + pageSize + totalElements + } + teasers { # Array + mediumTitle + links { target { id href title } } + type + } + }}''' % (client, playlist_id, pageNumber), + }).encode() + else: # mode == 'sammlung' + graphQL = json.dumps({ + 'query': '''{ + morePage( + client: "%s" + compilationId: "%s" + pageNumber: %d + ) { + widget { + pagination { + pageSize + totalElements + } + teasers { # Array + mediumTitle + links { target { id href title } } + type + } + } + }}''' % (client, playlist_id, pageNumber), + }).encode() + # Ressources for ARD graphQL debugging: + # https://api-test.ardmediathek.de/public-gateway + show_page = self._download_json( + 'https://api.ardmediathek.de/public-gateway', + '[Playlist] %s' % display_id, + data=graphQL, + headers={'Content-Type': 'application/json'})['data'] + # align the structure of the returned data: + if mode == 'sendung': + show_page = show_page['showPage'] + else: # mode == 'sammlung' + show_page = show_page['morePage']['widget'] + return show_page + + def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode): + """ Collects all playlist entries and returns them as info dict. + Supports playlists of mode 'sendung' and 'sammlung', and also nested + playlists. """ + entries = [] + pageNumber = 0 + while True: # iterate by pageNumber + show_page = self._ARD_load_playlist_snipped( + playlist_id, display_id, client, mode, pageNumber) + for teaser in show_page['teasers']: # process playlist items + if '/compilation/' in teaser['links']['target']['href']: + # alternativ cond.: teaser['type'] == "compilation" + # => This is an nested compilation, e.g. like: + # https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/ + link_mode = 'sammlung' + else: + link_mode = 'video' + + item_url = 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % ( + client, link_mode, display_id, + # perform HTLM quoting of episode title similar to ARD: + re.sub('^-|-$', '', # remove '-' from begin/end + re.sub('[^a-zA-Z0-9]+', '-', # replace special chars by - + teaser['links']['target']['title'].lower() + .replace('ä', 'ae').replace('ö', 'oe') + .replace('ü', 'ue').replace('ß', 'ss'))), + teaser['links']['target']['id']) + entries.append(self.url_result( + item_url, + ie=ARDBetaMediathekIE.ie_key())) + + if (show_page['pagination']['pageSize'] * (pageNumber + 1) + >= show_page['pagination']['totalElements']): + # we've processed enough pages to get all playlist entries + break + pageNumber = pageNumber + 1 + + return self.playlist_result(entries, playlist_title=display_id) + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('video_id') @@ -354,6 +498,13 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE): if not display_id: display_id = video_id + if mobj.group('mode') in ('sendung', 'sammlung'): + # this is a playlist-URL + return self._ARD_extract_playlist( + url, video_id, display_id, + mobj.group('client'), + mobj.group('mode')) + player_page = self._download_json( 'https://api.ardmediathek.de/public-gateway', display_id, data=json.dumps({ @@ -419,4 +570,5 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE): 'timestamp': unified_timestamp(player_page.get('broadcastedOn')), 'series': try_get(player_page, lambda x: x['show']['title']), }) + info.update(self._ARD_extract_episode_info(info['title'])) return info diff --git a/youtube_dl/extractor/arkena.py b/youtube_dlc/extractor/arkena.py index 854f58767..854f58767 100644 --- a/youtube_dl/extractor/arkena.py +++ b/youtube_dlc/extractor/arkena.py diff --git a/youtube_dl/extractor/arte.py b/youtube_dlc/extractor/arte.py index 2bd3bfe8a..2bd3bfe8a 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dlc/extractor/arte.py diff --git a/youtube_dl/extractor/asiancrush.py b/youtube_dlc/extractor/asiancrush.py index 0348e680c..0348e680c 100644 --- a/youtube_dl/extractor/asiancrush.py +++ b/youtube_dlc/extractor/asiancrush.py diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dlc/extractor/atresplayer.py index c2cec9845..c2cec9845 100644 --- a/youtube_dl/extractor/atresplayer.py +++ b/youtube_dlc/extractor/atresplayer.py diff --git a/youtube_dl/extractor/atttechchannel.py b/youtube_dlc/extractor/atttechchannel.py index 8f93fb353..8f93fb353 100644 --- a/youtube_dl/extractor/atttechchannel.py +++ b/youtube_dlc/extractor/atttechchannel.py diff --git a/youtube_dl/extractor/atvat.py b/youtube_dlc/extractor/atvat.py index 95e572d70..95e572d70 100644 --- a/youtube_dl/extractor/atvat.py +++ b/youtube_dlc/extractor/atvat.py diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dlc/extractor/audimedia.py index 6bd48ef15..6bd48ef15 100644 --- a/youtube_dl/extractor/audimedia.py +++ b/youtube_dlc/extractor/audimedia.py diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dlc/extractor/audioboom.py index c51837b40..c51837b40 100644 --- a/youtube_dl/extractor/audioboom.py +++ b/youtube_dlc/extractor/audioboom.py diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dlc/extractor/audiomack.py index cc7771354..cc7771354 100644 --- a/youtube_dl/extractor/audiomack.py +++ b/youtube_dlc/extractor/audiomack.py diff --git a/youtube_dl/extractor/awaan.py b/youtube_dlc/extractor/awaan.py index a2603bbff..a2603bbff 100644 --- a/youtube_dl/extractor/awaan.py +++ b/youtube_dlc/extractor/awaan.py diff --git a/youtube_dl/extractor/aws.py b/youtube_dlc/extractor/aws.py index dccfeaf73..dccfeaf73 100644 --- a/youtube_dl/extractor/aws.py +++ b/youtube_dlc/extractor/aws.py diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dlc/extractor/azmedien.py index b1e20def5..b1e20def5 100644 --- a/youtube_dl/extractor/azmedien.py +++ b/youtube_dlc/extractor/azmedien.py diff --git a/youtube_dl/extractor/baidu.py b/youtube_dlc/extractor/baidu.py index 234a661d3..234a661d3 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dlc/extractor/baidu.py diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dlc/extractor/bandcamp.py index f14b407dc..9dbafe86d 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dlc/extractor/bandcamp.py @@ -28,19 +28,22 @@ from ..utils import ( class BandcampIE(InfoExtractor): _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<title>[^/?#&]+)' _TESTS = [{ - 'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song', + 'url': 'http://youtube-dlc.bandcamp.com/track/youtube-dlc-test-song', 'md5': 'c557841d5e50261777a6585648adf439', 'info_dict': { 'id': '1812978515', 'ext': 'mp3', - 'title': "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad", + 'title': "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad", 'duration': 9.8485, + 'uploader': "youtube-dl \"'/\\\u00e4\u21ad", + 'timestamp': 1354224127, + 'upload_date': '20121129', }, '_skip': 'There is a limit of 200 free downloads / month for the test song' }, { # free download 'url': 'http://benprunty.bandcamp.com/track/lanius-battle', - 'md5': '853e35bf34aa1d6fe2615ae612564b36', + 'md5': '5d92af55811e47f38962a54c30b07ef0', 'info_dict': { 'id': '2650410135', 'ext': 'aiff', @@ -91,10 +94,11 @@ class BandcampIE(InfoExtractor): duration = None formats = [] - track_info = self._parse_json( - self._search_regex( - r'trackinfo\s*:\s*\[\s*({.+?})\s*\]\s*,\s*?\n', - webpage, 'track info', default='{}'), title) + trackinfo_block = self._html_search_regex( + r'trackinfo(?:["\']|"):\[\s*({.+?})\s*\],(?:["\']|")', + webpage, 'track info', default='{}') + + track_info = self._parse_json(trackinfo_block, title) if track_info: file_ = track_info.get('file') if isinstance(file_, dict): @@ -110,16 +114,18 @@ class BandcampIE(InfoExtractor): 'acodec': ext, 'abr': int_or_none(abr_str), }) - track = track_info.get('title') + track_id = str_or_none(track_info.get('track_id') or track_info.get('id')) track_number = int_or_none(track_info.get('track_num')) duration = float_or_none(track_info.get('duration')) def extract(key): - return self._search_regex( - r'\b%s\s*["\']?\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1' % key, + data = self._html_search_regex( + r',(["\']|")%s\1:\1(?P<value>(?:\\\1|((?!\1).))+)\1' % key, webpage, key, default=None, group='value') + return data.replace(r'\"', '"').replace('\\\\', '\\') if data else data + track = extract('title') artist = extract('artist') album = extract('album_title') timestamp = unified_timestamp( @@ -127,12 +133,12 @@ class BandcampIE(InfoExtractor): release_date = unified_strdate(extract('album_release_date')) download_link = self._search_regex( - r'freeDownloadPage\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage, + r'freeDownloadPage(?:["\']|"):\s*(["\']|")(?P<url>(?:(?!\1).)+)\1', webpage, 'download link', default=None, group='url') if download_link: track_id = self._search_regex( - r'(?ms)var TralbumData = .*?[{,]\s*id: (?P<id>\d+),?$', - webpage, 'track id') + r'\?id=(?P<id>\d+)&', + download_link, 'track id') download_webpage = self._download_webpage( download_link, track_id, 'Downloading free downloads page') @@ -315,10 +321,12 @@ class BandcampAlbumIE(InfoExtractor): if self._html_search_meta('duration', elem_content, default=None)] title = self._html_search_regex( - r'album_title\s*:\s*"((?:\\.|[^"\\])+?)"', - webpage, 'title', fatal=False) + r'album_title\s*(?:"|["\']):\s*("|["\'])(?P<album>(?:\\\1|((?!\1).))+)\1', + webpage, 'title', fatal=False, group='album') + if title: title = title.replace(r'\"', '"') + return { '_type': 'playlist', 'uploader_id': uploader_id, diff --git a/youtube_dl/extractor/bbc.py b/youtube_dlc/extractor/bbc.py index 002c39c39..002c39c39 100644 --- a/youtube_dl/extractor/bbc.py +++ b/youtube_dlc/extractor/bbc.py diff --git a/youtube_dl/extractor/beampro.py b/youtube_dlc/extractor/beampro.py index 86abdae00..86abdae00 100644 --- a/youtube_dl/extractor/beampro.py +++ b/youtube_dlc/extractor/beampro.py diff --git a/youtube_dl/extractor/beatport.py b/youtube_dlc/extractor/beatport.py index e60709417..e60709417 100644 --- a/youtube_dl/extractor/beatport.py +++ b/youtube_dlc/extractor/beatport.py diff --git a/youtube_dl/extractor/beeg.py b/youtube_dlc/extractor/beeg.py index 5788d13ba..5788d13ba 100644 --- a/youtube_dl/extractor/beeg.py +++ b/youtube_dlc/extractor/beeg.py diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dlc/extractor/behindkink.py index 9bca853b3..9bca853b3 100644 --- a/youtube_dl/extractor/behindkink.py +++ b/youtube_dlc/extractor/behindkink.py diff --git a/youtube_dl/extractor/bellmedia.py b/youtube_dlc/extractor/bellmedia.py index 9f9de96c6..9f9de96c6 100644 --- a/youtube_dl/extractor/bellmedia.py +++ b/youtube_dlc/extractor/bellmedia.py diff --git a/youtube_dl/extractor/bet.py b/youtube_dlc/extractor/bet.py index d7ceaa85e..2c7144235 100644 --- a/youtube_dl/extractor/bet.py +++ b/youtube_dlc/extractor/bet.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals from .mtv import MTVServicesInfoExtractor from ..utils import unified_strdate +# TODO Remove - Reason: Outdated Site + class BetIE(MTVServicesInfoExtractor): _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html' diff --git a/youtube_dl/extractor/bfi.py b/youtube_dlc/extractor/bfi.py index 60c8944b5..60c8944b5 100644 --- a/youtube_dl/extractor/bfi.py +++ b/youtube_dlc/extractor/bfi.py diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dlc/extractor/bigflix.py index 28e3e59f6..28e3e59f6 100644 --- a/youtube_dl/extractor/bigflix.py +++ b/youtube_dlc/extractor/bigflix.py diff --git a/youtube_dl/extractor/bild.py b/youtube_dlc/extractor/bild.py index b8dfbd42b..b8dfbd42b 100644 --- a/youtube_dl/extractor/bild.py +++ b/youtube_dlc/extractor/bild.py diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dlc/extractor/bilibili.py index 4dc597e16..d39ee8ffe 100644 --- a/youtube_dl/extractor/bilibili.py +++ b/youtube_dlc/extractor/bilibili.py @@ -139,7 +139,7 @@ class BiliBiliIE(InfoExtractor): webpage, 'player parameters'))['cid'][0] else: if 'no_bangumi_tip' not in smuggled_data: - self.to_screen('Downloading episode %s. To download all videos in anime %s, re-run youtube-dl with %s' % ( + self.to_screen('Downloading episode %s. To download all videos in anime %s, re-run youtube-dlc with %s' % ( video_id, anime_id, compat_urlparse.urljoin(url, '//bangumi.bilibili.com/anime/%s' % anime_id))) headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', diff --git a/youtube_dl/extractor/biobiochiletv.py b/youtube_dlc/extractor/biobiochiletv.py index dc86c57c5..dc86c57c5 100644 --- a/youtube_dl/extractor/biobiochiletv.py +++ b/youtube_dlc/extractor/biobiochiletv.py diff --git a/youtube_dl/extractor/biqle.py b/youtube_dlc/extractor/biqle.py index 17ebbb257..17ebbb257 100644 --- a/youtube_dl/extractor/biqle.py +++ b/youtube_dlc/extractor/biqle.py diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dlc/extractor/bitchute.py index 0c773e66e..92fc70b5a 100644 --- a/youtube_dl/extractor/bitchute.py +++ b/youtube_dlc/extractor/bitchute.py @@ -6,6 +6,8 @@ import re from .common import InfoExtractor from ..utils import ( + ExtractorError, + GeoRestrictedError, orderedSet, unified_strdate, urlencode_postdata, @@ -59,8 +61,14 @@ class BitChuteIE(InfoExtractor): for format_url in orderedSet(format_urls)] if not formats: - formats = self._parse_html5_media_entries( - url, webpage, video_id)[0]['formats'] + entries = self._parse_html5_media_entries( + url, webpage, video_id) + if not entries: + error = self._html_search_regex(r'<h1 class="page-title">([^<]+)</h1>', webpage, 'error', default='Cannot find video') + if error == 'Video Unavailable': + raise GeoRestrictedError(error) + raise ExtractorError(error) + formats = entries[0]['formats'] self._check_formats(formats, video_id) self._sort_formats(formats) diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dlc/extractor/bleacherreport.py index dc60224d0..dc60224d0 100644 --- a/youtube_dl/extractor/bleacherreport.py +++ b/youtube_dlc/extractor/bleacherreport.py diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dlc/extractor/blinkx.py index db5e12b21..db5e12b21 100644 --- a/youtube_dl/extractor/blinkx.py +++ b/youtube_dlc/extractor/blinkx.py diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dlc/extractor/bloomberg.py index 2fbfad1ba..2fbfad1ba 100644 --- a/youtube_dl/extractor/bloomberg.py +++ b/youtube_dlc/extractor/bloomberg.py diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dlc/extractor/bokecc.py index 6017e8344..6017e8344 100644 --- a/youtube_dl/extractor/bokecc.py +++ b/youtube_dlc/extractor/bokecc.py diff --git a/youtube_dl/extractor/bostonglobe.py b/youtube_dlc/extractor/bostonglobe.py index 57882fbee..57882fbee 100644 --- a/youtube_dl/extractor/bostonglobe.py +++ b/youtube_dlc/extractor/bostonglobe.py diff --git a/youtube_dl/extractor/bpb.py b/youtube_dlc/extractor/bpb.py index 07833532e..07833532e 100644 --- a/youtube_dl/extractor/bpb.py +++ b/youtube_dlc/extractor/bpb.py diff --git a/youtube_dl/extractor/br.py b/youtube_dlc/extractor/br.py index 9bde7f2d8..9bde7f2d8 100644 --- a/youtube_dl/extractor/br.py +++ b/youtube_dlc/extractor/br.py diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dlc/extractor/bravotv.py index b9715df00..b9715df00 100644 --- a/youtube_dl/extractor/bravotv.py +++ b/youtube_dlc/extractor/bravotv.py diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dlc/extractor/breakcom.py index 68c7cf2bb..68c7cf2bb 100644 --- a/youtube_dl/extractor/breakcom.py +++ b/youtube_dlc/extractor/breakcom.py diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dlc/extractor/brightcove.py index 2aa9f4782..2aa9f4782 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dlc/extractor/brightcove.py diff --git a/youtube_dl/extractor/businessinsider.py b/youtube_dlc/extractor/businessinsider.py index 73a57b1e4..73a57b1e4 100644 --- a/youtube_dl/extractor/businessinsider.py +++ b/youtube_dlc/extractor/businessinsider.py diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dlc/extractor/buzzfeed.py index ec411091e..ec411091e 100644 --- a/youtube_dl/extractor/buzzfeed.py +++ b/youtube_dlc/extractor/buzzfeed.py diff --git a/youtube_dl/extractor/byutv.py b/youtube_dlc/extractor/byutv.py index 0b11bf11f..0b11bf11f 100644 --- a/youtube_dl/extractor/byutv.py +++ b/youtube_dlc/extractor/byutv.py diff --git a/youtube_dl/extractor/c56.py b/youtube_dlc/extractor/c56.py index cac8fdcba..cac8fdcba 100644 --- a/youtube_dl/extractor/c56.py +++ b/youtube_dlc/extractor/c56.py diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dlc/extractor/camdemy.py index 8f0c6c545..8f0c6c545 100644 --- a/youtube_dl/extractor/camdemy.py +++ b/youtube_dlc/extractor/camdemy.py diff --git a/youtube_dl/extractor/cammodels.py b/youtube_dlc/extractor/cammodels.py index 1eb81b75e..1eb81b75e 100644 --- a/youtube_dl/extractor/cammodels.py +++ b/youtube_dlc/extractor/cammodels.py diff --git a/youtube_dl/extractor/camtube.py b/youtube_dlc/extractor/camtube.py index b3be3bdcf..b3be3bdcf 100644 --- a/youtube_dl/extractor/camtube.py +++ b/youtube_dlc/extractor/camtube.py diff --git a/youtube_dl/extractor/camwithher.py b/youtube_dlc/extractor/camwithher.py index bbc5205fd..bbc5205fd 100644 --- a/youtube_dl/extractor/camwithher.py +++ b/youtube_dlc/extractor/camwithher.py diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dlc/extractor/canalc2.py index 407cc8084..407cc8084 100644 --- a/youtube_dl/extractor/canalc2.py +++ b/youtube_dlc/extractor/canalc2.py diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dlc/extractor/canalplus.py index 51c11cb7e..51c11cb7e 100644 --- a/youtube_dl/extractor/canalplus.py +++ b/youtube_dlc/extractor/canalplus.py diff --git a/youtube_dl/extractor/canvas.py b/youtube_dlc/extractor/canvas.py index 8667a0d04..8667a0d04 100644 --- a/youtube_dl/extractor/canvas.py +++ b/youtube_dlc/extractor/canvas.py diff --git a/youtube_dl/extractor/carambatv.py b/youtube_dlc/extractor/carambatv.py index b57b86af7..b57b86af7 100644 --- a/youtube_dl/extractor/carambatv.py +++ b/youtube_dlc/extractor/carambatv.py diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dlc/extractor/cartoonnetwork.py index 48b33617f..48b33617f 100644 --- a/youtube_dl/extractor/cartoonnetwork.py +++ b/youtube_dlc/extractor/cartoonnetwork.py diff --git a/youtube_dl/extractor/cbc.py b/youtube_dlc/extractor/cbc.py index fd5ec6033..fd5ec6033 100644 --- a/youtube_dl/extractor/cbc.py +++ b/youtube_dlc/extractor/cbc.py diff --git a/youtube_dl/extractor/cbs.py b/youtube_dlc/extractor/cbs.py index 4a19a73d2..4a19a73d2 100644 --- a/youtube_dl/extractor/cbs.py +++ b/youtube_dlc/extractor/cbs.py diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dlc/extractor/cbsinteractive.py index 6596e98a6..6596e98a6 100644 --- a/youtube_dl/extractor/cbsinteractive.py +++ b/youtube_dlc/extractor/cbsinteractive.py diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dlc/extractor/cbslocal.py index 90852a9ef..90852a9ef 100644 --- a/youtube_dl/extractor/cbslocal.py +++ b/youtube_dlc/extractor/cbslocal.py diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dlc/extractor/cbsnews.py index 345debcf0..345debcf0 100644 --- a/youtube_dl/extractor/cbsnews.py +++ b/youtube_dlc/extractor/cbsnews.py diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dlc/extractor/cbssports.py index 83b764762..83b764762 100644 --- a/youtube_dl/extractor/cbssports.py +++ b/youtube_dlc/extractor/cbssports.py diff --git a/youtube_dl/extractor/ccc.py b/youtube_dlc/extractor/ccc.py index 36e6dff72..36e6dff72 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dlc/extractor/ccc.py diff --git a/youtube_dl/extractor/ccma.py b/youtube_dlc/extractor/ccma.py index 544647f92..544647f92 100644 --- a/youtube_dl/extractor/ccma.py +++ b/youtube_dlc/extractor/ccma.py diff --git a/youtube_dl/extractor/cctv.py b/youtube_dlc/extractor/cctv.py index c76f361c6..c76f361c6 100644 --- a/youtube_dl/extractor/cctv.py +++ b/youtube_dlc/extractor/cctv.py diff --git a/youtube_dl/extractor/cda.py b/youtube_dlc/extractor/cda.py index 0c3af23d5..0c3af23d5 100644 --- a/youtube_dl/extractor/cda.py +++ b/youtube_dlc/extractor/cda.py diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dlc/extractor/ceskatelevize.py index 7cb4efb74..7cb4efb74 100644 --- a/youtube_dl/extractor/ceskatelevize.py +++ b/youtube_dlc/extractor/ceskatelevize.py diff --git a/youtube_dl/extractor/channel9.py b/youtube_dlc/extractor/channel9.py index 09cacf6d3..09cacf6d3 100644 --- a/youtube_dl/extractor/channel9.py +++ b/youtube_dlc/extractor/channel9.py diff --git a/youtube_dl/extractor/charlierose.py b/youtube_dlc/extractor/charlierose.py index 42c9af263..42c9af263 100644 --- a/youtube_dl/extractor/charlierose.py +++ b/youtube_dlc/extractor/charlierose.py diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dlc/extractor/chaturbate.py index a459dcb8d..a459dcb8d 100644 --- a/youtube_dl/extractor/chaturbate.py +++ b/youtube_dlc/extractor/chaturbate.py diff --git a/youtube_dl/extractor/chilloutzone.py b/youtube_dlc/extractor/chilloutzone.py index 5aac21299..5aac21299 100644 --- a/youtube_dl/extractor/chilloutzone.py +++ b/youtube_dlc/extractor/chilloutzone.py diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dlc/extractor/chirbit.py index 8d75cdf19..8d75cdf19 100644 --- a/youtube_dl/extractor/chirbit.py +++ b/youtube_dlc/extractor/chirbit.py diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dlc/extractor/cinchcast.py index b861d54b0..b861d54b0 100644 --- a/youtube_dl/extractor/cinchcast.py +++ b/youtube_dlc/extractor/cinchcast.py diff --git a/youtube_dl/extractor/cinemax.py b/youtube_dlc/extractor/cinemax.py index 7f89d33de..7f89d33de 100644 --- a/youtube_dl/extractor/cinemax.py +++ b/youtube_dlc/extractor/cinemax.py diff --git a/youtube_dl/extractor/ciscolive.py b/youtube_dlc/extractor/ciscolive.py index da404e4dc..da404e4dc 100644 --- a/youtube_dl/extractor/ciscolive.py +++ b/youtube_dlc/extractor/ciscolive.py diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dlc/extractor/cjsw.py index 505bdbe16..505bdbe16 100644 --- a/youtube_dl/extractor/cjsw.py +++ b/youtube_dlc/extractor/cjsw.py diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dlc/extractor/cliphunter.py index f2ca7a337..f2ca7a337 100644 --- a/youtube_dl/extractor/cliphunter.py +++ b/youtube_dlc/extractor/cliphunter.py diff --git a/youtube_dl/extractor/clippit.py b/youtube_dlc/extractor/clippit.py index a1a7a774c..a1a7a774c 100644 --- a/youtube_dl/extractor/clippit.py +++ b/youtube_dlc/extractor/clippit.py diff --git a/youtube_dl/extractor/cliprs.py b/youtube_dlc/extractor/cliprs.py index d55b26d59..d55b26d59 100644 --- a/youtube_dl/extractor/cliprs.py +++ b/youtube_dlc/extractor/cliprs.py diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dlc/extractor/clipsyndicate.py index 6cdb42f5a..6cdb42f5a 100644 --- a/youtube_dl/extractor/clipsyndicate.py +++ b/youtube_dlc/extractor/clipsyndicate.py diff --git a/youtube_dl/extractor/closertotruth.py b/youtube_dlc/extractor/closertotruth.py index 26243d52d..26243d52d 100644 --- a/youtube_dl/extractor/closertotruth.py +++ b/youtube_dlc/extractor/closertotruth.py diff --git a/youtube_dl/extractor/cloudflarestream.py b/youtube_dlc/extractor/cloudflarestream.py index 2fdcfbb3a..2fdcfbb3a 100644 --- a/youtube_dl/extractor/cloudflarestream.py +++ b/youtube_dlc/extractor/cloudflarestream.py diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dlc/extractor/cloudy.py index 85ca20ecc..85ca20ecc 100644 --- a/youtube_dl/extractor/cloudy.py +++ b/youtube_dlc/extractor/cloudy.py diff --git a/youtube_dl/extractor/clubic.py b/youtube_dlc/extractor/clubic.py index 98f9cb596..98f9cb596 100644 --- a/youtube_dl/extractor/clubic.py +++ b/youtube_dlc/extractor/clubic.py diff --git a/youtube_dl/extractor/clyp.py b/youtube_dlc/extractor/clyp.py index 06d04de13..06d04de13 100644 --- a/youtube_dl/extractor/clyp.py +++ b/youtube_dlc/extractor/clyp.py diff --git a/youtube_dl/extractor/cmt.py b/youtube_dlc/extractor/cmt.py index e701fbeab..a4ddb9160 100644 --- a/youtube_dl/extractor/cmt.py +++ b/youtube_dlc/extractor/cmt.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals from .mtv import MTVIE +# TODO Remove - Reason: Outdated Site + class CMTIE(MTVIE): IE_NAME = 'cmt.com' @@ -39,7 +41,7 @@ class CMTIE(MTVIE): 'only_matching': True, }] - def _extract_mgid(self, webpage): + def _extract_mgid(self, webpage, url): mgid = self._search_regex( r'MTVN\.VIDEO\.contentUri\s*=\s*([\'"])(?P<mgid>.+?)\1', webpage, 'mgid', group='mgid', default=None) @@ -50,5 +52,5 @@ class CMTIE(MTVIE): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - mgid = self._extract_mgid(webpage) + mgid = self._extract_mgid(webpage, url) return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid) diff --git a/youtube_dl/extractor/cnbc.py b/youtube_dlc/extractor/cnbc.py index 6889b0f40..6889b0f40 100644 --- a/youtube_dl/extractor/cnbc.py +++ b/youtube_dlc/extractor/cnbc.py diff --git a/youtube_dl/extractor/cnn.py b/youtube_dlc/extractor/cnn.py index 774b71055..774b71055 100644 --- a/youtube_dl/extractor/cnn.py +++ b/youtube_dlc/extractor/cnn.py diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dlc/extractor/comedycentral.py index d08b909a6..f54c4adeb 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dlc/extractor/comedycentral.py @@ -48,7 +48,7 @@ class ComedyCentralFullEpisodesIE(MTVServicesInfoExtractor): def _real_extract(self, url): playlist_id = self._match_id(url) webpage = self._download_webpage(url, playlist_id) - mgid = self._extract_triforce_mgid(webpage, data_zone='t2_lc_promo1') + mgid = self._extract_mgid(webpage, url, data_zone='t2_lc_promo1') videos_info = self._get_videos_info(mgid) return videos_info diff --git a/youtube_dl/extractor/common.py b/youtube_dlc/extractor/common.py index a61753b17..4b42d699f 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dlc/extractor/common.py @@ -10,6 +10,7 @@ import os import random import re import socket +import ssl import sys import time import math @@ -67,6 +68,7 @@ from ..utils import ( sanitized_Request, sanitize_filename, str_or_none, + str_to_int, strip_or_none, unescapeHTML, unified_strdate, @@ -269,7 +271,7 @@ class InfoExtractor(object): Set to "root" to indicate that this is a comment to the original video. age_limit: Age restriction for the video, as an integer (years) - webpage_url: The URL to the video webpage, if given to youtube-dl it + webpage_url: The URL to the video webpage, if given to youtube-dlc it should allow to get the same result again. (It will be set by YoutubeDL if it's missing) categories: A list of categories that the video falls in, for example @@ -623,9 +625,12 @@ class InfoExtractor(object): url_or_request = update_url_query(url_or_request, query) if data is not None or headers: url_or_request = sanitized_Request(url_or_request, data, headers) + exceptions = [compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error] + if hasattr(ssl, 'CertificateError'): + exceptions.append(ssl.CertificateError) try: return self._downloader.urlopen(url_or_request) - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + except tuple(exceptions) as err: if isinstance(err, compat_urllib_error.HTTPError): if self.__can_accept_status_code(err, expected_status): # Retain reference to error to prevent file object from @@ -1244,7 +1249,10 @@ class InfoExtractor(object): interaction_type = is_e.get('interactionType') if not isinstance(interaction_type, compat_str): continue - interaction_count = int_or_none(is_e.get('userInteractionCount')) + # For interaction count some sites provide string instead of + # an integer (as per spec) with non digit characters (e.g. ",") + # so extracting count with more relaxed str_to_int + interaction_count = str_to_int(is_e.get('userInteractionCount')) if interaction_count is None: continue count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1]) @@ -1264,6 +1272,7 @@ class InfoExtractor(object): 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')), 'duration': parse_duration(e.get('duration')), 'timestamp': unified_timestamp(e.get('uploadDate')), + 'uploader': str_or_none(e.get('author')), 'filesize': float_or_none(e.get('contentSize')), 'tbr': int_or_none(e.get('bitrate')), 'width': int_or_none(e.get('width')), @@ -1500,7 +1509,7 @@ class InfoExtractor(object): if not isinstance(manifest, compat_etree_Element) and not fatal: return [] - # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy + # currently youtube-dlc cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0') if akamai_pv is not None and ';' in akamai_pv.text: playerVerificationChallenge = akamai_pv.text.split(';')[0] @@ -2071,8 +2080,9 @@ class InfoExtractor(object): http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip 2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP """ - if mpd_doc.get('type') == 'dynamic': - return [] + if not self._downloader.params.get('dynamic_mpd'): + if mpd_doc.get('type') == 'dynamic': + return [] namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None) diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dlc/extractor/commonmistakes.py index 7e12499b1..933b89eb3 100644 --- a/youtube_dl/extractor/commonmistakes.py +++ b/youtube_dlc/extractor/commonmistakes.py @@ -22,12 +22,12 @@ class CommonMistakesIE(InfoExtractor): def _real_extract(self, url): msg = ( - 'You\'ve asked youtube-dl to download the URL "%s". ' + 'You\'ve asked youtube-dlc to download the URL "%s". ' 'That doesn\'t make any sense. ' 'Simply remove the parameter in your command or configuration.' ) % url if not self._downloader.params.get('verbose'): - msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.' + msg += ' Add -v to the command line to see what arguments and configuration youtube-dlc got.' raise ExtractorError(msg, expected=True) diff --git a/youtube_dl/extractor/commonprotocols.py b/youtube_dlc/extractor/commonprotocols.py index d98331a4e..d98331a4e 100644 --- a/youtube_dl/extractor/commonprotocols.py +++ b/youtube_dlc/extractor/commonprotocols.py diff --git a/youtube_dl/extractor/condenast.py b/youtube_dlc/extractor/condenast.py index ed278fefc..ed278fefc 100644 --- a/youtube_dl/extractor/condenast.py +++ b/youtube_dlc/extractor/condenast.py diff --git a/youtube_dl/extractor/contv.py b/youtube_dlc/extractor/contv.py index 84b462d40..84b462d40 100644 --- a/youtube_dl/extractor/contv.py +++ b/youtube_dlc/extractor/contv.py diff --git a/youtube_dl/extractor/corus.py b/youtube_dlc/extractor/corus.py index e11aadf14..e11aadf14 100644 --- a/youtube_dl/extractor/corus.py +++ b/youtube_dlc/extractor/corus.py diff --git a/youtube_dl/extractor/coub.py b/youtube_dlc/extractor/coub.py index 6ea03e65c..6ea03e65c 100644 --- a/youtube_dl/extractor/coub.py +++ b/youtube_dlc/extractor/coub.py diff --git a/youtube_dl/extractor/cracked.py b/youtube_dlc/extractor/cracked.py index f77a68ece..f77a68ece 100644 --- a/youtube_dl/extractor/cracked.py +++ b/youtube_dlc/extractor/cracked.py diff --git a/youtube_dl/extractor/crackle.py b/youtube_dlc/extractor/crackle.py index 49bf3a4f9..49bf3a4f9 100644 --- a/youtube_dl/extractor/crackle.py +++ b/youtube_dlc/extractor/crackle.py diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dlc/extractor/crooksandliars.py index 7fb782db7..7fb782db7 100644 --- a/youtube_dl/extractor/crooksandliars.py +++ b/youtube_dlc/extractor/crooksandliars.py diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dlc/extractor/crunchyroll.py index bc2d1fa8b..bc2d1fa8b 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dlc/extractor/crunchyroll.py diff --git a/youtube_dl/extractor/cspan.py b/youtube_dlc/extractor/cspan.py index 67d6df4b0..67d6df4b0 100644 --- a/youtube_dl/extractor/cspan.py +++ b/youtube_dlc/extractor/cspan.py diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dlc/extractor/ctsnews.py index 679f1d92e..679f1d92e 100644 --- a/youtube_dl/extractor/ctsnews.py +++ b/youtube_dlc/extractor/ctsnews.py diff --git a/youtube_dl/extractor/ctvnews.py b/youtube_dlc/extractor/ctvnews.py index 03f8cefb7..03f8cefb7 100644 --- a/youtube_dl/extractor/ctvnews.py +++ b/youtube_dlc/extractor/ctvnews.py diff --git a/youtube_dl/extractor/cultureunplugged.py b/youtube_dlc/extractor/cultureunplugged.py index bcdf27323..bcdf27323 100644 --- a/youtube_dl/extractor/cultureunplugged.py +++ b/youtube_dlc/extractor/cultureunplugged.py diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dlc/extractor/curiositystream.py index e4a7fca6c..e4a7fca6c 100644 --- a/youtube_dl/extractor/curiositystream.py +++ b/youtube_dlc/extractor/curiositystream.py diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dlc/extractor/cwtv.py index 73382431b..73382431b 100644 --- a/youtube_dl/extractor/cwtv.py +++ b/youtube_dlc/extractor/cwtv.py diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dlc/extractor/dailymail.py index 67b88fd56..67b88fd56 100644 --- a/youtube_dl/extractor/dailymail.py +++ b/youtube_dlc/extractor/dailymail.py diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dlc/extractor/dailymotion.py index b8529050c..b8529050c 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dlc/extractor/dailymotion.py diff --git a/youtube_dl/extractor/daum.py b/youtube_dlc/extractor/daum.py index 137095577..137095577 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dlc/extractor/daum.py diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dlc/extractor/dbtv.py index aaedf2e3d..aaedf2e3d 100644 --- a/youtube_dl/extractor/dbtv.py +++ b/youtube_dlc/extractor/dbtv.py diff --git a/youtube_dl/extractor/dctp.py b/youtube_dlc/extractor/dctp.py index e700f8d86..e700f8d86 100644 --- a/youtube_dl/extractor/dctp.py +++ b/youtube_dlc/extractor/dctp.py diff --git a/youtube_dlc/extractor/deezer.py b/youtube_dlc/extractor/deezer.py new file mode 100644 index 000000000..3031671c1 --- /dev/null +++ b/youtube_dlc/extractor/deezer.py @@ -0,0 +1,147 @@ +from __future__ import unicode_literals + +import json +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + int_or_none, + orderedSet, +) + + +class DeezerBaseInfoExtractor(InfoExtractor): + def get_data(self, url): + if not self._downloader.params.get('test'): + self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') + + mobj = re.match(self._VALID_URL, url) + data_id = mobj.group('id') + + webpage = self._download_webpage(url, data_id) + geoblocking_msg = self._html_search_regex( + r'<p class="soon-txt">(.*?)</p>', webpage, 'geoblocking message', + default=None) + if geoblocking_msg is not None: + raise ExtractorError( + 'Deezer said: %s' % geoblocking_msg, expected=True) + + data_json = self._search_regex( + (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*</script>', + r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), + webpage, 'data JSON') + data = json.loads(data_json) + return data_id, webpage, data + + +class DeezerPlaylistIE(DeezerBaseInfoExtractor): + _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?playlist/(?P<id>[0-9]+)' + _TEST = { + 'url': 'http://www.deezer.com/playlist/176747451', + 'info_dict': { + 'id': '176747451', + 'title': 'Best!', + 'uploader': 'anonymous', + 'thumbnail': r're:^https?://(e-)?cdns-images\.dzcdn\.net/images/cover/.*\.jpg$', + }, + 'playlist_count': 29, + } + + def _real_extract(self, url): + playlist_id, webpage, data = self.get_data(url) + + playlist_title = data.get('DATA', {}).get('TITLE') + playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') + playlist_thumbnail = self._search_regex( + r'<img id="naboo_playlist_image".*?src="([^"]+)"', webpage, + 'playlist thumbnail') + + entries = [] + for s in data.get('SONGS', {}).get('data'): + formats = [{ + 'format_id': 'preview', + 'url': s.get('MEDIA', [{}])[0].get('HREF'), + 'preference': -100, # Only the first 30 seconds + 'ext': 'mp3', + }] + self._sort_formats(formats) + artists = ', '.join( + orderedSet(a.get('ART_NAME') for a in s.get('ARTISTS'))) + entries.append({ + 'id': s.get('SNG_ID'), + 'duration': int_or_none(s.get('DURATION')), + 'title': '%s - %s' % (artists, s.get('SNG_TITLE')), + 'uploader': s.get('ART_NAME'), + 'uploader_id': s.get('ART_ID'), + 'age_limit': 16 if s.get('EXPLICIT_LYRICS') == '1' else 0, + 'formats': formats, + }) + + return { + '_type': 'playlist', + 'id': playlist_id, + 'title': playlist_title, + 'uploader': playlist_uploader, + 'thumbnail': playlist_thumbnail, + 'entries': entries, + } + + +class DeezerAlbumIE(DeezerBaseInfoExtractor): + _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?album/(?P<id>[0-9]+)' + _TEST = { + 'url': 'https://www.deezer.com/fr/album/67505622', + 'info_dict': { + 'id': '67505622', + 'title': 'Last Week', + 'uploader': 'Home Brew', + 'thumbnail': r're:^https?://(e-)?cdns-images\.dzcdn\.net/images/cover/.*\.jpg$', + }, + 'playlist_count': 7, + } + + def _real_extract(self, url): + album_id, webpage, data = self.get_data(url) + + album_title = data.get('DATA', {}).get('ALB_TITLE') + album_uploader = data.get('DATA', {}).get('ART_NAME') + album_thumbnail = self._search_regex( + r'<img id="naboo_album_image".*?src="([^"]+)"', webpage, + 'album thumbnail') + + entries = [] + for s in data.get('SONGS', {}).get('data'): + formats = [{ + 'format_id': 'preview', + 'url': s.get('MEDIA', [{}])[0].get('HREF'), + 'preference': -100, # Only the first 30 seconds + 'ext': 'mp3', + }] + self._sort_formats(formats) + artists = ', '.join( + orderedSet(a.get('ART_NAME') for a in s.get('ARTISTS'))) + entries.append({ + 'id': s.get('SNG_ID'), + 'duration': int_or_none(s.get('DURATION')), + 'title': '%s - %s' % (artists, s.get('SNG_TITLE')), + 'uploader': s.get('ART_NAME'), + 'uploader_id': s.get('ART_ID'), + 'age_limit': 16 if s.get('EXPLICIT_LYRICS') == '1' else 0, + 'formats': formats, + 'track': s.get('SNG_TITLE'), + 'track_number': int_or_none(s.get('TRACK_NUMBER')), + 'track_id': s.get('SNG_ID'), + 'artist': album_uploader, + 'album': album_title, + 'album_artist': album_uploader, + }) + + return { + '_type': 'playlist', + 'id': album_id, + 'title': album_title, + 'uploader': album_uploader, + 'thumbnail': album_thumbnail, + 'entries': entries, + } diff --git a/youtube_dl/extractor/defense.py b/youtube_dlc/extractor/defense.py index 9fe144e14..9fe144e14 100644 --- a/youtube_dl/extractor/defense.py +++ b/youtube_dlc/extractor/defense.py diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dlc/extractor/democracynow.py index 5c9c0ecdc..5c9c0ecdc 100644 --- a/youtube_dl/extractor/democracynow.py +++ b/youtube_dlc/extractor/democracynow.py diff --git a/youtube_dl/extractor/dfb.py b/youtube_dlc/extractor/dfb.py index a4d0448c2..a4d0448c2 100644 --- a/youtube_dl/extractor/dfb.py +++ b/youtube_dlc/extractor/dfb.py diff --git a/youtube_dl/extractor/dhm.py b/youtube_dlc/extractor/dhm.py index aee72a6ed..aee72a6ed 100644 --- a/youtube_dl/extractor/dhm.py +++ b/youtube_dlc/extractor/dhm.py diff --git a/youtube_dl/extractor/digg.py b/youtube_dlc/extractor/digg.py index 913c1750f..913c1750f 100644 --- a/youtube_dl/extractor/digg.py +++ b/youtube_dlc/extractor/digg.py diff --git a/youtube_dl/extractor/digiteka.py b/youtube_dlc/extractor/digiteka.py index 3dfde0d8c..3dfde0d8c 100644 --- a/youtube_dl/extractor/digiteka.py +++ b/youtube_dlc/extractor/digiteka.py diff --git a/youtube_dl/extractor/discovery.py b/youtube_dlc/extractor/discovery.py index e0139cc86..e0139cc86 100644 --- a/youtube_dl/extractor/discovery.py +++ b/youtube_dlc/extractor/discovery.py diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dlc/extractor/discoverygo.py index 9e7b14a7d..9e7b14a7d 100644 --- a/youtube_dl/extractor/discoverygo.py +++ b/youtube_dlc/extractor/discoverygo.py diff --git a/youtube_dl/extractor/discoverynetworks.py b/youtube_dlc/extractor/discoverynetworks.py index 607a54948..607a54948 100644 --- a/youtube_dl/extractor/discoverynetworks.py +++ b/youtube_dlc/extractor/discoverynetworks.py diff --git a/youtube_dl/extractor/discoveryvr.py b/youtube_dlc/extractor/discoveryvr.py index cb63c2649..cb63c2649 100644 --- a/youtube_dl/extractor/discoveryvr.py +++ b/youtube_dlc/extractor/discoveryvr.py diff --git a/youtube_dl/extractor/disney.py b/youtube_dlc/extractor/disney.py index 0eee82fd6..0eee82fd6 100644 --- a/youtube_dl/extractor/disney.py +++ b/youtube_dlc/extractor/disney.py diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dlc/extractor/dispeak.py index c345e0274..22bdc5635 100644 --- a/youtube_dl/extractor/dispeak.py +++ b/youtube_dlc/extractor/dispeak.py @@ -93,6 +93,7 @@ class DigitallySpeakingIE(InfoExtractor): 'quality': -2, 'preference': -2, 'format_id': 'slides', + 'acodec': 'none', }) speaker_video_path = xpath_text(metadata, './speakerVideo', fatal=True) formats.append({ diff --git a/youtube_dl/extractor/dlive.py b/youtube_dlc/extractor/dlive.py index d95c67a5b..d95c67a5b 100644 --- a/youtube_dl/extractor/dlive.py +++ b/youtube_dlc/extractor/dlive.py diff --git a/youtube_dlc/extractor/doodstream.py b/youtube_dlc/extractor/doodstream.py new file mode 100644 index 000000000..2c9ea6898 --- /dev/null +++ b/youtube_dlc/extractor/doodstream.py @@ -0,0 +1,71 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import string +import random +import time + +from .common import InfoExtractor + + +class DoodStreamIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?dood\.(?:to|watch)/[ed]/(?P<id>[a-z0-9]+)' + _TESTS = [{ + 'url': 'http://dood.to/e/5s1wmbdacezb', + 'md5': '4568b83b31e13242b3f1ff96c55f0595', + 'info_dict': { + 'id': '5s1wmbdacezb', + 'ext': 'mp4', + 'title': 'Kat Wonders - Monthly May 2020', + 'description': 'Kat Wonders - Monthly May 2020 | DoodStream.com', + 'thumbnail': 'https://img.doodcdn.com/snaps/flyus84qgl2fsk4g.jpg', + } + }, { + 'url': 'https://dood.to/d/jzrxn12t2s7n', + 'md5': '3207e199426eca7c2aa23c2872e6728a', + 'info_dict': { + 'id': 'jzrxn12t2s7n', + 'ext': 'mp4', + 'title': 'Stacy Cruz Cute ALLWAYSWELL', + 'description': 'Stacy Cruz Cute ALLWAYSWELL | DoodStream.com', + 'thumbnail': 'https://img.doodcdn.com/snaps/8edqd5nppkac3x8u.jpg', + } + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + if '/d/' in url: + url = "https://dood.to" + self._html_search_regex( + r'<iframe src="(/e/[a-z0-9]+)"', webpage, 'embed') + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_meta(['og:title', 'twitter:title'], + webpage, default=None) + thumb = self._html_search_meta(['og:image', 'twitter:image'], + webpage, default=None) + token = self._html_search_regex(r'[?&]token=([a-z0-9]+)[&\']', webpage, 'token') + description = self._html_search_meta( + ['og:description', 'description', 'twitter:description'], + webpage, default=None) + auth_url = 'https://dood.to' + self._html_search_regex( + r'(/pass_md5.*?)\'', webpage, 'pass_md5') + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/66.0', + 'referer': url + } + + webpage = self._download_webpage(auth_url, video_id, headers=headers) + final_url = webpage + ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(10)]) + "?token=" + token + "&expiry=" + str(int(time.time() * 1000)) + + return { + 'id': video_id, + 'title': title, + 'url': final_url, + 'http_headers': headers, + 'ext': 'mp4', + 'description': description, + 'thumbnail': thumb, + } diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dlc/extractor/dotsub.py index 148605c0b..148605c0b 100644 --- a/youtube_dl/extractor/dotsub.py +++ b/youtube_dlc/extractor/dotsub.py diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dlc/extractor/douyutv.py index 9757f4422..9757f4422 100644 --- a/youtube_dl/extractor/douyutv.py +++ b/youtube_dlc/extractor/douyutv.py diff --git a/youtube_dl/extractor/dplay.py b/youtube_dlc/extractor/dplay.py index a7b9db568..a7b9db568 100644 --- a/youtube_dl/extractor/dplay.py +++ b/youtube_dlc/extractor/dplay.py diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dlc/extractor/drbonanza.py index 164e97c36..164e97c36 100644 --- a/youtube_dl/extractor/drbonanza.py +++ b/youtube_dlc/extractor/drbonanza.py diff --git a/youtube_dl/extractor/dropbox.py b/youtube_dlc/extractor/dropbox.py index 14b6c00b0..9dc6614c5 100644 --- a/youtube_dl/extractor/dropbox.py +++ b/youtube_dlc/extractor/dropbox.py @@ -13,11 +13,11 @@ class DropboxIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/sh?/(?P<id>[a-zA-Z0-9]{15})/.*' _TESTS = [ { - 'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0', + 'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dlc%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0', 'info_dict': { 'id': 'nelirfsxnmcfbfh', 'ext': 'mp4', - 'title': 'youtube-dl test video \'ä"BaW_jenozKc' + 'title': 'youtube-dlc test video \'ä"BaW_jenozKc' } }, { 'url': 'https://www.dropbox.com/sh/662glsejgzoj9sr/AAByil3FGH9KFNZ13e08eSa1a/Pregame%20Ceremony%20Program%20PA%2020140518.m4v', diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dlc/extractor/drtuber.py index 2baea585b..2baea585b 100644 --- a/youtube_dl/extractor/drtuber.py +++ b/youtube_dlc/extractor/drtuber.py diff --git a/youtube_dl/extractor/drtv.py b/youtube_dlc/extractor/drtv.py index 390e79f8c..390e79f8c 100644 --- a/youtube_dl/extractor/drtv.py +++ b/youtube_dlc/extractor/drtv.py diff --git a/youtube_dl/extractor/dtube.py b/youtube_dlc/extractor/dtube.py index 114d2dbe3..114d2dbe3 100644 --- a/youtube_dl/extractor/dtube.py +++ b/youtube_dlc/extractor/dtube.py diff --git a/youtube_dlc/extractor/duboku.py b/youtube_dlc/extractor/duboku.py new file mode 100644 index 000000000..fdc695bf4 --- /dev/null +++ b/youtube_dlc/extractor/duboku.py @@ -0,0 +1,242 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import compat_urlparse +from ..utils import ( + clean_html, + extract_attributes, + ExtractorError, + get_elements_by_class, + int_or_none, + js_to_json, + smuggle_url, + unescapeHTML, +) + + +def _get_elements_by_tag_and_attrib(html, tag=None, attribute=None, value=None, escape_value=True): + """Return the content of the tag with the specified attribute in the passed HTML document""" + + if tag is None: + tag = '[a-zA-Z0-9:._-]+' + if attribute is None: + attribute = '' + else: + attribute = r'\s+(?P<attribute>%s)' % re.escape(attribute) + if value is None: + value = '' + else: + value = re.escape(value) if escape_value else value + value = '=[\'"]?(?P<value>%s)[\'"]?' % value + + retlist = [] + for m in re.finditer(r'''(?xs) + <(?P<tag>%s) + (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*? + %s%s + (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*? + \s*> + (?P<content>.*?) + </\1> + ''' % (tag, attribute, value), html): + retlist.append(m) + + return retlist + + +def _get_element_by_tag_and_attrib(html, tag=None, attribute=None, value=None, escape_value=True): + retval = _get_elements_by_tag_and_attrib(html, tag, attribute, value, escape_value) + return retval[0] if retval else None + + +class DubokuIE(InfoExtractor): + IE_NAME = 'duboku' + IE_DESC = 'www.duboku.co' + + _VALID_URL = r'(?:https?://[^/]+\.duboku\.co/vodplay/)(?P<id>[0-9]+-[0-9-]+)\.html.*' + _TESTS = [{ + 'url': 'https://www.duboku.co/vodplay/1575-1-1.html', + 'info_dict': { + 'id': '1575-1-1', + 'ext': 'ts', + 'series': '白色月光', + 'title': 'contains:白色月光', + 'season_number': 1, + 'episode_number': 1, + }, + 'params': { + 'skip_download': 'm3u8 download', + }, + }, { + 'url': 'https://www.duboku.co/vodplay/1588-1-1.html', + 'info_dict': { + 'id': '1588-1-1', + 'ext': 'ts', + 'series': '亲爱的自己', + 'title': 'contains:预告片', + 'season_number': 1, + 'episode_number': 1, + }, + 'params': { + 'skip_download': 'm3u8 download', + }, + }] + + _PLAYER_DATA_PATTERN = r'player_data\s*=\s*(\{\s*(.*)})\s*;?\s*</script' + + def _real_extract(self, url): + video_id = self._match_id(url) + temp = video_id.split('-') + series_id = temp[0] + season_id = temp[1] + episode_id = temp[2] + + webpage_url = 'https://www.duboku.co/vodplay/%s.html' % video_id + webpage_html = self._download_webpage(webpage_url, video_id) + + # extract video url + + player_data = self._search_regex( + self._PLAYER_DATA_PATTERN, webpage_html, 'player_data') + player_data = self._parse_json(player_data, video_id, js_to_json) + + # extract title + + temp = get_elements_by_class('title', webpage_html) + series_title = None + title = None + for html in temp: + mobj = re.search(r'<a\s+.*>(.*)</a>', html) + if mobj: + href = extract_attributes(mobj.group(0)).get('href') + if href: + mobj1 = re.search(r'/(\d+)\.html', href) + if mobj1 and mobj1.group(1) == series_id: + series_title = clean_html(mobj.group(0)) + series_title = re.sub(r'[\s\r\n\t]+', ' ', series_title) + title = clean_html(html) + title = re.sub(r'[\s\r\n\t]+', ' ', title) + break + + data_url = player_data.get('url') + if not data_url: + raise ExtractorError('Cannot find url in player_data') + data_from = player_data.get('from') + + # if it is an embedded iframe, maybe it's an external source + if data_from == 'iframe': + # use _type url_transparent to retain the meaningful details + # of the video. + return { + '_type': 'url_transparent', + 'url': smuggle_url(data_url, {'http_headers': {'Referer': webpage_url}}), + 'id': video_id, + 'title': title, + 'series': series_title, + 'season_number': int_or_none(season_id), + 'season_id': season_id, + 'episode_number': int_or_none(episode_id), + 'episode_id': episode_id, + } + + formats = self._extract_m3u8_formats(data_url, video_id, 'mp4') + + return { + 'id': video_id, + 'title': title, + 'series': series_title, + 'season_number': int_or_none(season_id), + 'season_id': season_id, + 'episode_number': int_or_none(episode_id), + 'episode_id': episode_id, + 'formats': formats, + 'http_headers': {'Referer': 'https://www.duboku.co/static/player/videojs.html'} + } + + +class DubokuPlaylistIE(InfoExtractor): + IE_NAME = 'duboku:list' + IE_DESC = 'www.duboku.co entire series' + + _VALID_URL = r'(?:https?://[^/]+\.duboku\.co/voddetail/)(?P<id>[0-9]+)\.html.*' + _TESTS = [{ + 'url': 'https://www.duboku.co/voddetail/1575.html', + 'info_dict': { + 'id': 'startswith:1575', + 'title': '白色月光', + }, + 'playlist_count': 12, + }, { + 'url': 'https://www.duboku.co/voddetail/1554.html', + 'info_dict': { + 'id': 'startswith:1554', + 'title': '以家人之名', + }, + 'playlist_mincount': 30, + }, { + 'url': 'https://www.duboku.co/voddetail/1554.html#playlist2', + 'info_dict': { + 'id': '1554#playlist2', + 'title': '以家人之名', + }, + 'playlist_mincount': 27, + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError('Invalid URL: %s' % url) + series_id = mobj.group('id') + fragment = compat_urlparse.urlparse(url).fragment + + webpage_url = 'https://www.duboku.co/voddetail/%s.html' % series_id + webpage_html = self._download_webpage(webpage_url, series_id) + + # extract title + + title = _get_element_by_tag_and_attrib(webpage_html, 'h1', 'class', 'title') + title = unescapeHTML(title.group('content')) if title else None + if not title: + title = self._html_search_meta('keywords', webpage_html) + if not title: + title = _get_element_by_tag_and_attrib(webpage_html, 'title') + title = unescapeHTML(title.group('content')) if title else None + + # extract playlists + + playlists = {} + for div in _get_elements_by_tag_and_attrib( + webpage_html, attribute='id', value='playlist\\d+', escape_value=False): + playlist_id = div.group('value') + playlist = [] + for a in _get_elements_by_tag_and_attrib( + div.group('content'), 'a', 'href', value='[^\'"]+?', escape_value=False): + playlist.append({ + 'href': unescapeHTML(a.group('value')), + 'title': unescapeHTML(a.group('content')) + }) + playlists[playlist_id] = playlist + + # select the specified playlist if url fragment exists + playlist = None + playlist_id = None + if fragment: + playlist = playlists.get(fragment) + playlist_id = fragment + else: + first = next(iter(playlists.items()), None) + if first: + (playlist_id, playlist) = first + if not playlist: + raise ExtractorError( + 'Cannot find %s' % fragment if fragment else 'Cannot extract playlist') + + # return url results + return self.playlist_result([ + self.url_result( + compat_urlparse.urljoin('https://www.duboku.co', x['href']), + ie=DubokuIE.ie_key(), video_title=x.get('title')) + for x in playlist], series_id + '#' + playlist_id, title) diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dlc/extractor/dumpert.py index d9d9afdec..d9d9afdec 100644 --- a/youtube_dl/extractor/dumpert.py +++ b/youtube_dlc/extractor/dumpert.py diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dlc/extractor/dvtv.py index de7f6d670..de7f6d670 100644 --- a/youtube_dl/extractor/dvtv.py +++ b/youtube_dlc/extractor/dvtv.py diff --git a/youtube_dl/extractor/dw.py b/youtube_dlc/extractor/dw.py index d740652f1..d740652f1 100644 --- a/youtube_dl/extractor/dw.py +++ b/youtube_dlc/extractor/dw.py diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dlc/extractor/eagleplatform.py index 36fef07b7..36fef07b7 100644 --- a/youtube_dl/extractor/eagleplatform.py +++ b/youtube_dlc/extractor/eagleplatform.py diff --git a/youtube_dl/extractor/ebaumsworld.py b/youtube_dlc/extractor/ebaumsworld.py index c97682cd3..c97682cd3 100644 --- a/youtube_dl/extractor/ebaumsworld.py +++ b/youtube_dlc/extractor/ebaumsworld.py diff --git a/youtube_dl/extractor/echomsk.py b/youtube_dlc/extractor/echomsk.py index 6b7cc652f..6b7cc652f 100644 --- a/youtube_dl/extractor/echomsk.py +++ b/youtube_dlc/extractor/echomsk.py diff --git a/youtube_dl/extractor/egghead.py b/youtube_dlc/extractor/egghead.py index df11dc206..df11dc206 100644 --- a/youtube_dl/extractor/egghead.py +++ b/youtube_dlc/extractor/egghead.py diff --git a/youtube_dl/extractor/ehow.py b/youtube_dlc/extractor/ehow.py index b1cd4f5d4..b1cd4f5d4 100644 --- a/youtube_dl/extractor/ehow.py +++ b/youtube_dlc/extractor/ehow.py diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dlc/extractor/eighttracks.py index 9a44f89f3..5ededd31d 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dlc/extractor/eighttracks.py @@ -18,12 +18,12 @@ class EightTracksIE(InfoExtractor): _VALID_URL = r'https?://8tracks\.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$' _TEST = { 'name': 'EightTracks', - 'url': 'http://8tracks.com/ytdl/youtube-dl-test-tracks-a', + 'url': 'http://8tracks.com/ytdl/youtube-dlc-test-tracks-a', 'info_dict': { 'id': '1336550', - 'display_id': 'youtube-dl-test-tracks-a', + 'display_id': 'youtube-dlc-test-tracks-a', 'description': "test chars: \"'/\\ä↭", - 'title': "youtube-dl test tracks \"'/\\ä↭<>", + 'title': "youtube-dlc test tracks \"'/\\ä↭<>", }, 'playlist': [ { @@ -31,7 +31,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885610', 'ext': 'm4a', - 'title': "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad", + 'title': "youtue-dl project<>\"' - youtube-dlc test track 1 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } }, @@ -40,7 +40,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885608', 'ext': 'm4a', - 'title': "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad", + 'title': "youtube-dlc project - youtube-dlc test track 2 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } }, @@ -49,7 +49,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885679', 'ext': 'm4a', - 'title': "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad", + 'title': "youtube-dlc project as well - youtube-dlc test track 3 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } }, @@ -58,7 +58,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885680', 'ext': 'm4a', - 'title': "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad", + 'title': "youtube-dlc project as well - youtube-dlc test track 4 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } }, @@ -67,7 +67,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885682', 'ext': 'm4a', - 'title': "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad", + 'title': "PH - youtube-dlc test track 5 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } }, @@ -76,7 +76,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885683', 'ext': 'm4a', - 'title': "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad", + 'title': "PH - youtube-dlc test track 6 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } }, @@ -85,7 +85,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885684', 'ext': 'm4a', - 'title': "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad", + 'title': "phihag - youtube-dlc test track 7 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } }, @@ -94,7 +94,7 @@ class EightTracksIE(InfoExtractor): 'info_dict': { 'id': '11885685', 'ext': 'm4a', - 'title': "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad", + 'title': "phihag - youtube-dlc test track 8 \"'/\\\u00e4\u21ad", 'uploader_id': 'ytdl' } } diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dlc/extractor/einthusan.py index 4e0f8bc81..4e0f8bc81 100644 --- a/youtube_dl/extractor/einthusan.py +++ b/youtube_dlc/extractor/einthusan.py diff --git a/youtube_dl/extractor/eitb.py b/youtube_dlc/extractor/eitb.py index ee5ead18b..ee5ead18b 100644 --- a/youtube_dl/extractor/eitb.py +++ b/youtube_dlc/extractor/eitb.py diff --git a/youtube_dl/extractor/ellentube.py b/youtube_dlc/extractor/ellentube.py index 544473274..544473274 100644 --- a/youtube_dl/extractor/ellentube.py +++ b/youtube_dlc/extractor/ellentube.py diff --git a/youtube_dlc/extractor/elonet.py b/youtube_dlc/extractor/elonet.py new file mode 100644 index 000000000..c64ab5de3 --- /dev/null +++ b/youtube_dlc/extractor/elonet.py @@ -0,0 +1,137 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import os +import re +import tempfile + +from .common import InfoExtractor +from ..utils import ( + base_url, + ExtractorError, + try_get, +) +from ..compat import compat_str +from ..downloader.hls import HlsFD + + +class ElonetIE(InfoExtractor): + _VALID_URL = r'https?://elonet\.finna\.fi/Record/kavi\.elonet_elokuva_(?P<id>[0-9]+)' + _TEST = { + 'url': 'https://elonet.finna.fi/Record/kavi.elonet_elokuva_107867', + 'md5': '8efc954b96c543711707f87de757caea', + 'info_dict': { + 'id': '107867', + 'ext': 'mp4', + 'title': 'Valkoinen peura', + 'description': 'Valkoinen peura (1952) on Erik Blombergin ohjaama ja yhdessä Mirjami Kuosmasen kanssa käsikirjoittama tarunomainen kertomus valkoisen peuran hahmossa lii...', + 'thumbnail': 'https://elonet.finna.fi/Cover/Show?id=kavi.elonet_elokuva_107867&index=0&size=large', + }, + } + + def _download_m3u8_chunked_subtitle(self, chunklist_url): + """ + Download VTT subtitles from pieces in manifest URL. + Return a string containing joined chunks with extra headers removed. + """ + with tempfile.NamedTemporaryFile(delete=True) as outfile: + fname = outfile.name + hlsdl = HlsFD(self._downloader, {}) + hlsdl.download(compat_str(fname), {"url": chunklist_url}) + with open(fname, 'r') as fin: + # Remove (some) headers + fdata = re.sub(r'X-TIMESTAMP-MAP.*\n+|WEBVTT\n+', '', fin.read()) + os.remove(fname) + return "WEBVTT\n\n" + fdata + + def _parse_m3u8_subtitles(self, m3u8_doc, m3u8_url): + """ + Parse subtitles from HLS / m3u8 manifest. + """ + subtitles = {} + baseurl = m3u8_url[:m3u8_url.rindex('/') + 1] + for line in m3u8_doc.split('\n'): + if 'EXT-X-MEDIA:TYPE=SUBTITLES' in line: + lang = self._search_regex( + r'LANGUAGE="(.+?)"', line, 'lang', default=False) + uri = self._search_regex( + r'URI="(.+?)"', line, 'uri', default=False) + if lang and uri: + data = self._download_m3u8_chunked_subtitle(baseurl + uri) + subtitles[lang] = [{'ext': 'vtt', 'data': data}] + return subtitles + + def _parse_mpd_subtitles(self, mpd_doc): + """ + Parse subtitles from MPD manifest. + """ + ns = '{urn:mpeg:dash:schema:mpd:2011}' + subtitles = {} + for aset in mpd_doc.findall(".//%sAdaptationSet[@mimeType='text/vtt']" % (ns)): + lang = aset.attrib.get('lang', 'unk') + url = aset.find("./%sRepresentation/%sBaseURL" % (ns, ns)).text + subtitles[lang] = [{'ext': 'vtt', 'url': url}] + return subtitles + + def _get_subtitles(self, fmt, doc, url): + if fmt == 'm3u8': + subs = self._parse_m3u8_subtitles(doc, url) + elif fmt == 'mpd': + subs = self._parse_mpd_subtitles(doc) + else: + self._downloader.report_warning( + "Cannot download subtitles from '%s' streams." % (fmt)) + subs = {} + return subs + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex( + r'<meta .*property="og:title" .*content="(.+?)"', webpage, 'title') + description = self._html_search_regex( + r'<meta .*property="og:description" .*content="(.+?)"', webpage, 'description') + thumbnail = self._html_search_regex( + r'<meta .*property="og:image" .*content="(.+?)"', webpage, 'thumbnail') + + json_s = self._html_search_regex( + r'data-video-sources="(.+?)"', webpage, 'json') + src = try_get( + self._parse_json(json_s, video_id), + lambda x: x[0]["src"], compat_str) + formats = [] + if re.search(r'\.m3u8\??', src): + fmt = 'm3u8' + res = self._download_webpage_handle( + # elonet servers have certificate problems + src.replace('https:', 'http:'), video_id, + note='Downloading m3u8 information', + errnote='Failed to download m3u8 information') + if res: + doc, urlh = res + url = urlh.geturl() + formats = self._parse_m3u8_formats(doc, url) + for f in formats: + f['ext'] = 'mp4' + elif re.search(r'\.mpd\??', src): + fmt = 'mpd' + res = self._download_xml_handle( + src, video_id, + note='Downloading MPD manifest', + errnote='Failed to download MPD manifest') + if res: + doc, urlh = res + url = base_url(urlh.geturl()) + formats = self._parse_mpd_formats(doc, mpd_base_url=url) + else: + raise ExtractorError("Unknown streaming format") + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'formats': formats, + 'subtitles': self.extract_subtitles(fmt, doc, url), + } diff --git a/youtube_dl/extractor/elpais.py b/youtube_dlc/extractor/elpais.py index b89f6db62..b89f6db62 100644 --- a/youtube_dl/extractor/elpais.py +++ b/youtube_dlc/extractor/elpais.py diff --git a/youtube_dl/extractor/embedly.py b/youtube_dlc/extractor/embedly.py index a5820b21e..a5820b21e 100644 --- a/youtube_dl/extractor/embedly.py +++ b/youtube_dlc/extractor/embedly.py diff --git a/youtube_dl/extractor/engadget.py b/youtube_dlc/extractor/engadget.py index 65635c18b..65635c18b 100644 --- a/youtube_dl/extractor/engadget.py +++ b/youtube_dlc/extractor/engadget.py diff --git a/youtube_dl/extractor/eporner.py b/youtube_dlc/extractor/eporner.py index fe42821c7..fe42821c7 100644 --- a/youtube_dl/extractor/eporner.py +++ b/youtube_dlc/extractor/eporner.py diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dlc/extractor/eroprofile.py index c08643a17..c08643a17 100644 --- a/youtube_dl/extractor/eroprofile.py +++ b/youtube_dlc/extractor/eroprofile.py diff --git a/youtube_dl/extractor/escapist.py b/youtube_dlc/extractor/escapist.py index 4cd815ebc..4cd815ebc 100644 --- a/youtube_dl/extractor/escapist.py +++ b/youtube_dlc/extractor/escapist.py diff --git a/youtube_dl/extractor/espn.py b/youtube_dlc/extractor/espn.py index 6cf05e6da..6cf05e6da 100644 --- a/youtube_dl/extractor/espn.py +++ b/youtube_dlc/extractor/espn.py diff --git a/youtube_dl/extractor/esri.py b/youtube_dlc/extractor/esri.py index e9dcaeb1d..e9dcaeb1d 100644 --- a/youtube_dl/extractor/esri.py +++ b/youtube_dlc/extractor/esri.py diff --git a/youtube_dl/extractor/europa.py b/youtube_dlc/extractor/europa.py index 1efc0b2ec..1efc0b2ec 100644 --- a/youtube_dl/extractor/europa.py +++ b/youtube_dlc/extractor/europa.py diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dlc/extractor/everyonesmixtape.py index 84a9b750e..84a9b750e 100644 --- a/youtube_dl/extractor/everyonesmixtape.py +++ b/youtube_dlc/extractor/everyonesmixtape.py diff --git a/youtube_dl/extractor/expotv.py b/youtube_dlc/extractor/expotv.py index 95a897782..95a897782 100644 --- a/youtube_dl/extractor/expotv.py +++ b/youtube_dlc/extractor/expotv.py diff --git a/youtube_dl/extractor/expressen.py b/youtube_dlc/extractor/expressen.py index f79365038..dc8b855d2 100644 --- a/youtube_dl/extractor/expressen.py +++ b/youtube_dlc/extractor/expressen.py @@ -15,7 +15,7 @@ from ..utils import ( class ExpressenIE(InfoExtractor): _VALID_URL = r'''(?x) https?:// - (?:www\.)?expressen\.se/ + (?:www\.)?(?:expressen|di)\.se/ (?:(?:tvspelare/video|videoplayer/embed)/)? tv/(?:[^/]+/)* (?P<id>[^/?#&]+) @@ -42,13 +42,16 @@ class ExpressenIE(InfoExtractor): }, { 'url': 'https://www.expressen.se/videoplayer/embed/tv/ditv/ekonomistudion/experterna-har-ar-fragorna-som-avgor-valet/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di', 'only_matching': True, + }, { + 'url': 'https://www.di.se/videoplayer/embed/tv/ditv/borsmorgon/implantica-rusar-70--under-borspremiaren-hor-styrelsemedlemmen/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di', + 'only_matching': True, }] @staticmethod def _extract_urls(webpage): return [ mobj.group('url') for mobj in re.finditer( - r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?expressen\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1', + r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?(?:expressen|di)\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1', webpage)] def _real_extract(self, url): diff --git a/youtube_dl/extractor/extractors.py b/youtube_dlc/extractor/extractors.py index 9564465a0..f0860e04d 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dlc/extractor/extractors.py @@ -36,6 +36,10 @@ from .afreecatv import AfreecaTVIE from .airmozilla import AirMozillaIE from .aljazeera import AlJazeeraIE from .alphaporno import AlphaPornoIE +from .alura import ( + AluraIE, + AluraCourseIE +) from .amcnetworks import AMCNetworksIE from .americastestkitchen import AmericasTestKitchenIE from .animeondemand import AnimeOnDemandIE @@ -262,7 +266,10 @@ from .daum import ( ) from .dbtv import DBTVIE from .dctp import DctpTvIE -from .deezer import DeezerPlaylistIE +from .deezer import ( + DeezerPlaylistIE, + DeezerAlbumIE, +) from .democracynow import DemocracynowIE from .dfb import DFBIE from .dhm import DHMIE @@ -273,7 +280,6 @@ from .douyutv import ( DouyuTVIE, ) from .dplay import DPlayIE -from .dreisat import DreiSatIE from .drbonanza import DRBonanzaIE from .drtuber import DrTuberIE from .drtv import ( @@ -282,6 +288,10 @@ from .drtv import ( ) from .dtube import DTubeIE from .dvtv import DVTVIE +from .duboku import ( + DubokuIE, + DubokuPlaylistIE +) from .dumpert import DumpertIE from .defense import DefenseGouvFrIE from .discovery import DiscoveryIE @@ -293,6 +303,7 @@ from .discoverynetworks import DiscoveryNetworksDeIE from .discoveryvr import DiscoveryVRIE from .disney import DisneyIE from .dispeak import DigitallySpeakingIE +from .doodstream import DoodStreamIE from .dropbox import DropboxIE from .dw import ( DWIE, @@ -314,6 +325,7 @@ from .ellentube import ( EllenTubeVideoIE, EllenTubePlaylistIE, ) +from .elonet import ElonetIE from .elpais import ElPaisIE from .embedly import EmbedlyIE from .engadget import EngadgetIE @@ -440,6 +452,7 @@ from .hotstar import ( ) from .howcast import HowcastIE from .howstuffworks import HowStuffWorksIE +from .hrfensehen import HRFernsehenIE from .hrti import ( HRTiIE, HRTiPlaylistIE, @@ -585,6 +598,7 @@ from .lynda import ( LyndaCourseIE ) from .m6 import M6IE +from .magentamusik360 import MagentaMusik360IE from .mailru import ( MailRuIE, MailRuMusicIE, @@ -667,12 +681,16 @@ from .myvi import ( MyviIE, MyviEmbedIE, ) +from .myvideoge import MyVideoGeIE from .myvidster import MyVidsterIE from .nationalgeographic import ( NationalGeographicVideoIE, NationalGeographicTVIE, ) -from .naver import NaverIE +from .naver import ( + NaverIE, + NaverLiveIE, +) from .nba import NBAIE from .nbc import ( CSNNEIE, @@ -858,7 +876,10 @@ from .pluralsight import ( PluralsightCourseIE, ) from .podomatic import PodomaticIE -from .pokemon import PokemonIE +from .pokemon import ( + PokemonIE, + PokemonWatchIE, +) from .polskieradio import ( PolskieRadioIE, PolskieRadioCategoryIE, @@ -918,7 +939,9 @@ from .rbmaradio import RBMARadioIE from .rds import RDSIE from .redbulltv import ( RedBullTVIE, + RedBullEmbedIE, RedBullTVRrnContentIE, + RedBullIE, ) from .reddit import ( RedditIE, @@ -1057,6 +1080,11 @@ from .spike import ( BellatorIE, ParamountNetworkIE, ) +from .storyfire import ( + StoryFireIE, + StoryFireUserIE, + StoryFireSeriesIE, +) from .stitcher import StitcherIE from .sport5 import Sport5IE from .sportbox import SportBoxIE @@ -1143,10 +1171,7 @@ from .thisamericanlife import ThisAmericanLifeIE from .thisav import ThisAVIE from .thisoldhouse import ThisOldHouseIE from .threeqsdn import ThreeQSDNIE -from .tiktok import ( - TikTokIE, - TikTokUserIE, -) +from .tiktok import TikTokIE from .tinypic import TinyPicIE from .tmz import ( TMZIE, @@ -1207,6 +1232,7 @@ from .tvnet import TVNetIE from .tvnoe import TVNoeIE from .tvnow import ( TVNowIE, + TVNowFilmIE, TVNowNewIE, TVNowSeasonIE, TVNowAnnualIE, diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dlc/extractor/extremetube.py index acd4090fa..acd4090fa 100644 --- a/youtube_dl/extractor/extremetube.py +++ b/youtube_dlc/extractor/extremetube.py diff --git a/youtube_dl/extractor/eyedotv.py b/youtube_dlc/extractor/eyedotv.py index f62ddebae..f62ddebae 100644 --- a/youtube_dl/extractor/eyedotv.py +++ b/youtube_dlc/extractor/eyedotv.py diff --git a/youtube_dl/extractor/facebook.py b/youtube_dlc/extractor/facebook.py index 610d66745..610d66745 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dlc/extractor/facebook.py diff --git a/youtube_dl/extractor/faz.py b/youtube_dlc/extractor/faz.py index 312ee2aee..312ee2aee 100644 --- a/youtube_dl/extractor/faz.py +++ b/youtube_dlc/extractor/faz.py diff --git a/youtube_dl/extractor/fc2.py b/youtube_dlc/extractor/fc2.py index 435561147..435561147 100644 --- a/youtube_dl/extractor/fc2.py +++ b/youtube_dlc/extractor/fc2.py diff --git a/youtube_dl/extractor/fczenit.py b/youtube_dlc/extractor/fczenit.py index 8db7c5963..8db7c5963 100644 --- a/youtube_dl/extractor/fczenit.py +++ b/youtube_dlc/extractor/fczenit.py diff --git a/youtube_dl/extractor/filmon.py b/youtube_dlc/extractor/filmon.py index f775fe0ba..f775fe0ba 100644 --- a/youtube_dl/extractor/filmon.py +++ b/youtube_dlc/extractor/filmon.py diff --git a/youtube_dl/extractor/filmweb.py b/youtube_dlc/extractor/filmweb.py index 56000bc5b..56000bc5b 100644 --- a/youtube_dl/extractor/filmweb.py +++ b/youtube_dlc/extractor/filmweb.py diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dlc/extractor/firsttv.py index 28617d83c..28617d83c 100644 --- a/youtube_dl/extractor/firsttv.py +++ b/youtube_dlc/extractor/firsttv.py diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dlc/extractor/fivemin.py index f3f876ecd..f3f876ecd 100644 --- a/youtube_dl/extractor/fivemin.py +++ b/youtube_dlc/extractor/fivemin.py diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dlc/extractor/fivetv.py index c4c0f1b3d..c4c0f1b3d 100644 --- a/youtube_dl/extractor/fivetv.py +++ b/youtube_dlc/extractor/fivetv.py diff --git a/youtube_dl/extractor/flickr.py b/youtube_dlc/extractor/flickr.py index 9f166efd4..9f166efd4 100644 --- a/youtube_dl/extractor/flickr.py +++ b/youtube_dlc/extractor/flickr.py diff --git a/youtube_dl/extractor/folketinget.py b/youtube_dlc/extractor/folketinget.py index b3df93f28..b3df93f28 100644 --- a/youtube_dl/extractor/folketinget.py +++ b/youtube_dlc/extractor/folketinget.py diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dlc/extractor/footyroom.py index 118325b6d..118325b6d 100644 --- a/youtube_dl/extractor/footyroom.py +++ b/youtube_dlc/extractor/footyroom.py diff --git a/youtube_dl/extractor/formula1.py b/youtube_dlc/extractor/formula1.py index fecfc28ae..fecfc28ae 100644 --- a/youtube_dl/extractor/formula1.py +++ b/youtube_dlc/extractor/formula1.py diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dlc/extractor/fourtube.py index be4e81342..be4e81342 100644 --- a/youtube_dl/extractor/fourtube.py +++ b/youtube_dlc/extractor/fourtube.py diff --git a/youtube_dl/extractor/fox.py b/youtube_dlc/extractor/fox.py index 04f4bdba6..04f4bdba6 100644 --- a/youtube_dl/extractor/fox.py +++ b/youtube_dlc/extractor/fox.py diff --git a/youtube_dl/extractor/fox9.py b/youtube_dlc/extractor/fox9.py index 91f8f7b8a..91f8f7b8a 100644 --- a/youtube_dl/extractor/fox9.py +++ b/youtube_dlc/extractor/fox9.py diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dlc/extractor/foxgay.py index 512a10645..512a10645 100644 --- a/youtube_dl/extractor/foxgay.py +++ b/youtube_dlc/extractor/foxgay.py diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dlc/extractor/foxnews.py index 63613cb85..63613cb85 100644 --- a/youtube_dl/extractor/foxnews.py +++ b/youtube_dlc/extractor/foxnews.py diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dlc/extractor/foxsports.py index 2b2cb6c6f..2b2cb6c6f 100644 --- a/youtube_dl/extractor/foxsports.py +++ b/youtube_dlc/extractor/foxsports.py diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dlc/extractor/franceculture.py index 306b45fc9..306b45fc9 100644 --- a/youtube_dl/extractor/franceculture.py +++ b/youtube_dlc/extractor/franceculture.py diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dlc/extractor/franceinter.py index 05806895c..05806895c 100644 --- a/youtube_dl/extractor/franceinter.py +++ b/youtube_dlc/extractor/franceinter.py diff --git a/youtube_dl/extractor/francetv.py b/youtube_dlc/extractor/francetv.py index 81b468c7d..e340cddba 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dlc/extractor/francetv.py @@ -316,13 +316,14 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor): _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/(?:[^/]+/)*(?P<id>[^/?#&.]+)' _TESTS = [{ - 'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html', + 'url': 'https://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-jeudi-22-aout-2019_3561461.html', 'info_dict': { - 'id': '84981923', + 'id': 'd12458ee-5062-48fe-bfdd-a30d6a01b793', 'ext': 'mp4', 'title': 'Soir 3', - 'upload_date': '20130826', - 'timestamp': 1377548400, + 'upload_date': '20190822', + 'timestamp': 1566510900, + 'description': 'md5:72d167097237701d6e8452ff03b83c00', 'subtitles': { 'fr': 'mincount:2', }, @@ -374,7 +375,8 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor): video_id = self._search_regex( (r'player\.load[^;]+src:\s*["\']([^"\']+)', r'id-video=([^@]+@[^"]+)', - r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"'), + r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"', + r'data-id="([^"]+)"'), webpage, 'video id') return self._make_url_result(video_id) diff --git a/youtube_dl/extractor/freesound.py b/youtube_dlc/extractor/freesound.py index 138b6bc58..138b6bc58 100644 --- a/youtube_dl/extractor/freesound.py +++ b/youtube_dlc/extractor/freesound.py diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dlc/extractor/freespeech.py index ea9c3e317..ea9c3e317 100644 --- a/youtube_dl/extractor/freespeech.py +++ b/youtube_dlc/extractor/freespeech.py diff --git a/youtube_dl/extractor/freshlive.py b/youtube_dlc/extractor/freshlive.py index 72a845945..72a845945 100644 --- a/youtube_dl/extractor/freshlive.py +++ b/youtube_dlc/extractor/freshlive.py diff --git a/youtube_dl/extractor/frontendmasters.py b/youtube_dlc/extractor/frontendmasters.py index f1db33fb1..f1db33fb1 100644 --- a/youtube_dl/extractor/frontendmasters.py +++ b/youtube_dlc/extractor/frontendmasters.py diff --git a/youtube_dl/extractor/funimation.py b/youtube_dlc/extractor/funimation.py index 8bbedca26..8bbedca26 100644 --- a/youtube_dl/extractor/funimation.py +++ b/youtube_dlc/extractor/funimation.py diff --git a/youtube_dl/extractor/funk.py b/youtube_dlc/extractor/funk.py index 81d1949fd..81d1949fd 100644 --- a/youtube_dl/extractor/funk.py +++ b/youtube_dlc/extractor/funk.py diff --git a/youtube_dl/extractor/fusion.py b/youtube_dlc/extractor/fusion.py index a3f44b812..a3f44b812 100644 --- a/youtube_dl/extractor/fusion.py +++ b/youtube_dlc/extractor/fusion.py diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dlc/extractor/fxnetworks.py index 00e67426b..00e67426b 100644 --- a/youtube_dl/extractor/fxnetworks.py +++ b/youtube_dlc/extractor/fxnetworks.py diff --git a/youtube_dl/extractor/gaia.py b/youtube_dlc/extractor/gaia.py index e9527758f..e9527758f 100644 --- a/youtube_dl/extractor/gaia.py +++ b/youtube_dlc/extractor/gaia.py diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dlc/extractor/gameinformer.py index f1b96c172..f1b96c172 100644 --- a/youtube_dl/extractor/gameinformer.py +++ b/youtube_dlc/extractor/gameinformer.py diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dlc/extractor/gamespot.py index 4236a5ed8..4236a5ed8 100644 --- a/youtube_dl/extractor/gamespot.py +++ b/youtube_dlc/extractor/gamespot.py diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dlc/extractor/gamestar.py index f00dab2f3..f00dab2f3 100644 --- a/youtube_dl/extractor/gamestar.py +++ b/youtube_dlc/extractor/gamestar.py diff --git a/youtube_dl/extractor/gaskrank.py b/youtube_dlc/extractor/gaskrank.py index 1726a6704..1726a6704 100644 --- a/youtube_dl/extractor/gaskrank.py +++ b/youtube_dlc/extractor/gaskrank.py diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dlc/extractor/gazeta.py index 57c67a451..57c67a451 100644 --- a/youtube_dl/extractor/gazeta.py +++ b/youtube_dlc/extractor/gazeta.py diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dlc/extractor/gdcvault.py index 2f555c1d4..a248a170d 100644 --- a/youtube_dl/extractor/gdcvault.py +++ b/youtube_dlc/extractor/gdcvault.py @@ -5,9 +5,7 @@ import re from .common import InfoExtractor from .kaltura import KalturaIE from ..utils import ( - HEADRequest, sanitized_Request, - smuggle_url, urlencode_postdata, ) @@ -122,67 +120,38 @@ class GDCVaultIE(InfoExtractor): request = sanitized_Request(login_url, urlencode_postdata(login_form)) request.add_header('Content-Type', 'application/x-www-form-urlencoded') self._download_webpage(request, display_id, 'Logging in') - start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page') + webpage = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page') self._download_webpage(logout_url, display_id, 'Logging out') - return start_page + return webpage def _real_extract(self, url): video_id, name = re.match(self._VALID_URL, url).groups() display_id = name or video_id - webpage_url = 'http://www.gdcvault.com/play/' + video_id - start_page = self._download_webpage(webpage_url, display_id) - - direct_url = self._search_regex( - r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);', - start_page, 'url', default=None) - if direct_url: - title = self._html_search_regex( - r'<td><strong>Session Name:?</strong></td>\s*<td>(.*?)</td>', - start_page, 'title') - video_url = 'http://www.gdcvault.com' + direct_url - # resolve the url so that we can detect the correct extension - video_url = self._request_webpage( - HEADRequest(video_url), video_id).geturl() - - return { - 'id': video_id, - 'display_id': display_id, - 'url': video_url, - 'title': title, - } + webpage = self._download_webpage(url, display_id) + + title = self._html_search_regex( + r'<td><strong>Session Name:?</strong></td>\s*<td>(.*?)</td>', + webpage, 'title') + + PLAYER_REGEX = r'<iframe src=\"(?P<manifest_url>.*?)\".*?</iframe>' + manifest_url = self._html_search_regex( + PLAYER_REGEX, webpage, 'manifest_url') + + partner_id = self._search_regex( + r'/p(?:artner_id)?/(\d+)', manifest_url, 'partner id', + default='1670711') - embed_url = KalturaIE._extract_url(start_page) - if embed_url: - embed_url = smuggle_url(embed_url, {'source_url': url}) - ie_key = 'Kaltura' - else: - PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/(?:gdc-)?player.*?\.html.*?".*?</iframe>' - - xml_root = self._html_search_regex( - PLAYER_REGEX, start_page, 'xml root', default=None) - if xml_root is None: - # Probably need to authenticate - login_res = self._login(webpage_url, display_id) - if login_res is None: - self.report_warning('Could not login.') - else: - start_page = login_res - # Grab the url from the authenticated page - xml_root = self._html_search_regex( - PLAYER_REGEX, start_page, 'xml root') - - xml_name = self._html_search_regex( - r'<iframe src=".*?\?xml(?:=|URL=xml/)(.+?\.xml).*?".*?</iframe>', - start_page, 'xml filename') - embed_url = '%s/xml/%s' % (xml_root, xml_name) - ie_key = 'DigitallySpeaking' + kaltura_id = self._search_regex( + r'entry_id=(?P<id>(?:[^&])+)', manifest_url, + 'kaltura id', group='id') return { '_type': 'url_transparent', + 'url': 'kaltura:%s:%s' % (partner_id, kaltura_id), + 'ie_key': KalturaIE.ie_key(), 'id': video_id, 'display_id': display_id, - 'url': embed_url, - 'ie_key': ie_key, + 'title': title, } diff --git a/youtube_dl/extractor/generic.py b/youtube_dlc/extractor/generic.py index 355067a50..aba06b328 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dlc/extractor/generic.py @@ -1947,7 +1947,7 @@ class GenericIE(InfoExtractor): }, { # vshare embed - 'url': 'https://youtube-dl-demo.neocities.org/vshare.html', + 'url': 'https://youtube-dlc-demo.neocities.org/vshare.html', 'md5': '17b39f55b5497ae8b59f5fbce8e35886', 'info_dict': { 'id': '0f64ce6', @@ -2263,7 +2263,7 @@ class GenericIE(InfoExtractor): if default_search == 'auto_warning': if re.match(r'^(?:url|URL)$', url): raise ExtractorError( - 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url, + 'Invalid URL: %r . Call youtube-dlc like this: youtube-dlc -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url, expected=True) else: self._downloader.report_warning( @@ -2273,7 +2273,7 @@ class GenericIE(InfoExtractor): if default_search in ('error', 'fixup_error'): raise ExtractorError( '%r is not a valid URL. ' - 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube' + 'Set --default-search "ytsearch" (or run youtube-dlc "ytsearch:%s" ) to search YouTube' % (url, url), expected=True) else: if ':' not in default_search: @@ -2349,7 +2349,7 @@ class GenericIE(InfoExtractor): request = sanitized_Request(url) # Some webservers may serve compressed content of rather big size (e.g. gzipped flac) # making it impossible to download only chunk of the file (yet we need only 512kB to - # test whether it's HTML or not). According to youtube-dl default Accept-Encoding + # test whether it's HTML or not). According to youtube-dlc default Accept-Encoding # that will always result in downloading the whole file that is not desirable. # Therefore for extraction pass we have to override Accept-Encoding to any in order # to accept raw bytes and being able to download only a chunk. @@ -3372,7 +3372,7 @@ class GenericIE(InfoExtractor): if not found: # twitter:player is a https URL to iframe player that may or may not - # be supported by youtube-dl thus this is checked the very last (see + # be supported by youtube-dlc thus this is checked the very last (see # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser) embed_url = self._html_search_meta('twitter:player', webpage, default=None) if embed_url and embed_url != url: diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dlc/extractor/gfycat.py index 18a30fe67..18a30fe67 100644 --- a/youtube_dl/extractor/gfycat.py +++ b/youtube_dlc/extractor/gfycat.py diff --git a/youtube_dl/extractor/giantbomb.py b/youtube_dlc/extractor/giantbomb.py index c6477958d..c6477958d 100644 --- a/youtube_dl/extractor/giantbomb.py +++ b/youtube_dlc/extractor/giantbomb.py diff --git a/youtube_dl/extractor/giga.py b/youtube_dlc/extractor/giga.py index 5a9992a27..5a9992a27 100644 --- a/youtube_dl/extractor/giga.py +++ b/youtube_dlc/extractor/giga.py diff --git a/youtube_dl/extractor/gigya.py b/youtube_dlc/extractor/gigya.py index 412178492..412178492 100644 --- a/youtube_dl/extractor/gigya.py +++ b/youtube_dlc/extractor/gigya.py diff --git a/youtube_dl/extractor/glide.py b/youtube_dlc/extractor/glide.py index d94dfbf09..d94dfbf09 100644 --- a/youtube_dl/extractor/glide.py +++ b/youtube_dlc/extractor/glide.py diff --git a/youtube_dl/extractor/globo.py b/youtube_dlc/extractor/globo.py index 60d842d3a..60d842d3a 100644 --- a/youtube_dl/extractor/globo.py +++ b/youtube_dlc/extractor/globo.py diff --git a/youtube_dl/extractor/go.py b/youtube_dlc/extractor/go.py index 03cfba91f..7a75dfa49 100644 --- a/youtube_dl/extractor/go.py +++ b/youtube_dlc/extractor/go.py @@ -137,7 +137,11 @@ class GoIE(AdobePassIE): # There may be inner quotes, e.g. data-video-id="'VDKA3609139'" # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood r'data-video-id=["\']*(VDKA\w+)', - # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet + # https://github.com/ytdl-org/youtube-dl/pull/25216/files + # The following is based on the pull request on the line above. Changed the ABC.com URL to a show available now. + # https://abc.com/shows/the-rookie/episode-guide/season-02/19-the-q-word + r'\bvideoIdCode["\']\s*:\s*["\'](vdka\w+)', + # Deprecated fallback pattern r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)' ), webpage, 'video id', default=video_id) if not site_info: diff --git a/youtube_dl/extractor/godtube.py b/youtube_dlc/extractor/godtube.py index 92efd16b3..92efd16b3 100644 --- a/youtube_dl/extractor/godtube.py +++ b/youtube_dlc/extractor/godtube.py diff --git a/youtube_dl/extractor/golem.py b/youtube_dlc/extractor/golem.py index 47a068e74..47a068e74 100644 --- a/youtube_dl/extractor/golem.py +++ b/youtube_dlc/extractor/golem.py diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dlc/extractor/googledrive.py index 589e4d5c3..ec0d58a57 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dlc/extractor/googledrive.py @@ -220,19 +220,27 @@ class GoogleDriveIE(InfoExtractor): 'id': video_id, 'export': 'download', }) - urlh = self._request_webpage( - source_url, video_id, note='Requesting source file', - errnote='Unable to request source file', fatal=False) + + def request_source_file(source_url, kind): + return self._request_webpage( + source_url, video_id, note='Requesting %s file' % kind, + errnote='Unable to request %s file' % kind, fatal=False) + urlh = request_source_file(source_url, 'source') if urlh: - def add_source_format(src_url): + def add_source_format(urlh): formats.append({ - 'url': src_url, + # Use redirect URLs as download URLs in order to calculate + # correct cookies in _calc_cookies. + # Using original URLs may result in redirect loop due to + # google.com's cookies mistakenly used for googleusercontent.com + # redirect URLs (see #23919). + 'url': urlh.geturl(), 'ext': determine_ext(title, 'mp4').lower(), 'format_id': 'source', 'quality': 1, }) if urlh.headers.get('Content-Disposition'): - add_source_format(source_url) + add_source_format(urlh) else: confirmation_webpage = self._webpage_read_content( urlh, url, video_id, note='Downloading confirmation page', @@ -242,9 +250,12 @@ class GoogleDriveIE(InfoExtractor): r'confirm=([^&"\']+)', confirmation_webpage, 'confirmation code', fatal=False) if confirm: - add_source_format(update_url_query(source_url, { + confirmed_source_url = update_url_query(source_url, { 'confirm': confirm, - })) + }) + urlh = request_source_file(confirmed_source_url, 'confirmed source') + if urlh and urlh.headers.get('Content-Disposition'): + add_source_format(urlh) if not formats: reason = self._search_regex( @@ -265,6 +276,8 @@ class GoogleDriveIE(InfoExtractor): subtitles_id = ttsurl.encode('utf-8').decode( 'unicode_escape').split('=')[-1] + self._downloader.cookiejar.clear(domain='.google.com', path='/', name='NID') + return { 'id': video_id, 'title': title, diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dlc/extractor/googleplus.py index 6b927bb44..6b927bb44 100644 --- a/youtube_dl/extractor/googleplus.py +++ b/youtube_dlc/extractor/googleplus.py diff --git a/youtube_dl/extractor/googlesearch.py b/youtube_dlc/extractor/googlesearch.py index 5279fa807..5279fa807 100644 --- a/youtube_dl/extractor/googlesearch.py +++ b/youtube_dlc/extractor/googlesearch.py diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dlc/extractor/goshgay.py index 377981d3e..377981d3e 100644 --- a/youtube_dl/extractor/goshgay.py +++ b/youtube_dlc/extractor/goshgay.py diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dlc/extractor/gputechconf.py index 73dc62c49..73dc62c49 100644 --- a/youtube_dl/extractor/gputechconf.py +++ b/youtube_dlc/extractor/gputechconf.py diff --git a/youtube_dl/extractor/groupon.py b/youtube_dlc/extractor/groupon.py index a6da90931..a6da90931 100644 --- a/youtube_dl/extractor/groupon.py +++ b/youtube_dlc/extractor/groupon.py diff --git a/youtube_dl/extractor/hbo.py b/youtube_dlc/extractor/hbo.py index 68df748f5..68df748f5 100644 --- a/youtube_dl/extractor/hbo.py +++ b/youtube_dlc/extractor/hbo.py diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dlc/extractor/hearthisat.py index 18c252012..18c252012 100644 --- a/youtube_dl/extractor/hearthisat.py +++ b/youtube_dlc/extractor/hearthisat.py diff --git a/youtube_dl/extractor/heise.py b/youtube_dlc/extractor/heise.py index cbe564a3c..cbe564a3c 100644 --- a/youtube_dl/extractor/heise.py +++ b/youtube_dlc/extractor/heise.py diff --git a/youtube_dl/extractor/hellporno.py b/youtube_dlc/extractor/hellporno.py index fae425103..fae425103 100644 --- a/youtube_dl/extractor/hellporno.py +++ b/youtube_dlc/extractor/hellporno.py diff --git a/youtube_dl/extractor/helsinki.py b/youtube_dlc/extractor/helsinki.py index 575fb332a..575fb332a 100644 --- a/youtube_dl/extractor/helsinki.py +++ b/youtube_dlc/extractor/helsinki.py diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dlc/extractor/hentaistigma.py index 86a93de4d..86a93de4d 100644 --- a/youtube_dl/extractor/hentaistigma.py +++ b/youtube_dlc/extractor/hentaistigma.py diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dlc/extractor/hgtv.py index a4f332565..a4f332565 100644 --- a/youtube_dl/extractor/hgtv.py +++ b/youtube_dlc/extractor/hgtv.py diff --git a/youtube_dl/extractor/hidive.py b/youtube_dlc/extractor/hidive.py index f26f80265..f26f80265 100644 --- a/youtube_dl/extractor/hidive.py +++ b/youtube_dlc/extractor/hidive.py diff --git a/youtube_dl/extractor/historicfilms.py b/youtube_dlc/extractor/historicfilms.py index 56343e98f..56343e98f 100644 --- a/youtube_dl/extractor/historicfilms.py +++ b/youtube_dlc/extractor/historicfilms.py diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dlc/extractor/hitbox.py index 3e5ff2685..3e5ff2685 100644 --- a/youtube_dl/extractor/hitbox.py +++ b/youtube_dlc/extractor/hitbox.py diff --git a/youtube_dl/extractor/hitrecord.py b/youtube_dlc/extractor/hitrecord.py index fd5dc2935..fd5dc2935 100644 --- a/youtube_dl/extractor/hitrecord.py +++ b/youtube_dlc/extractor/hitrecord.py diff --git a/youtube_dl/extractor/hketv.py b/youtube_dlc/extractor/hketv.py index 1f3502b90..1f3502b90 100644 --- a/youtube_dl/extractor/hketv.py +++ b/youtube_dlc/extractor/hketv.py diff --git a/youtube_dl/extractor/hornbunny.py b/youtube_dlc/extractor/hornbunny.py index c458a959d..c458a959d 100644 --- a/youtube_dl/extractor/hornbunny.py +++ b/youtube_dlc/extractor/hornbunny.py diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dlc/extractor/hotnewhiphop.py index 4703e1894..4703e1894 100644 --- a/youtube_dl/extractor/hotnewhiphop.py +++ b/youtube_dlc/extractor/hotnewhiphop.py diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dlc/extractor/hotstar.py index f97eefa3d..1fb4d2d41 100644 --- a/youtube_dl/extractor/hotstar.py +++ b/youtube_dlc/extractor/hotstar.py @@ -6,11 +6,13 @@ import hmac import re import time import uuid +import json +import random from .common import InfoExtractor from ..compat import ( compat_HTTPError, - compat_str, + compat_str ) from ..utils import ( determine_ext, @@ -30,16 +32,42 @@ class HotStarBaseIE(InfoExtractor): exp = st + 6000 auth = 'st=%d~exp=%d~acl=/*' % (st, exp) auth += '~hmac=' + hmac.new(self._AKAMAI_ENCRYPTION_KEY, auth.encode(), hashlib.sha256).hexdigest() + + def _generate_device_id(): + """ + Reversed from javascript library. + JS function is generateUUID + """ + t = int(round(time.time() * 1000)) + e = "xxxxxxxx-xxxx-4xxx-xxxx-xxxxxxxxxxxx" # 4 seems to be interchangeable + + def _replacer(): + n = int((t + 16 * random.random())) % 16 | 0 + return hex(n if "x" == e else 3 & n | 8)[2:] + return "".join([_.replace('x', _replacer()) for _ in e]) + + token = self._download_json( + 'https://api.hotstar.com/um/v3/users', + video_id, note='Downloading token', + data=json.dumps({"device_ids": [{"id": compat_str(uuid.uuid4()), "type": "device_id"}]}).encode('utf-8'), + headers={ + 'hotstarauth': auth, + 'x-hs-platform': 'PCTV', # or 'web' + 'Content-Type': 'application/json', + })['user_identity'] + response = self._download_json( 'https://api.hotstar.com/' + path, video_id, headers={ 'hotstarauth': auth, - 'x-country-code': 'IN', - 'x-platform-code': 'JIO', + 'x-hs-appversion': '6.72.2', + 'x-hs-platform': 'web', + 'x-hs-usertoken': token, }, query=query) - if response['statusCode'] != 'OK': + + if response['message'] != "Playback URL's fetched successfully": raise ExtractorError( - response['body']['message'], expected=True) - return response['body']['results'] + response['message'], expected=True) + return response['data'] def _call_api(self, path, video_id, query_name='contentId'): return self._call_api_impl(path, video_id, { @@ -49,19 +77,17 @@ class HotStarBaseIE(InfoExtractor): def _call_api_v2(self, path, video_id): return self._call_api_impl( - '%s/in/contents/%s' % (path, video_id), video_id, { - 'desiredConfig': 'encryption:plain;ladder:phone,tv;package:hls,dash', - 'client': 'mweb', - 'clientVersion': '6.18.0', - 'deviceId': compat_str(uuid.uuid4()), - 'osName': 'Windows', - 'osVersion': '10', + '%s/content/%s' % (path, video_id), video_id, { + 'desired-config': 'audio_channel:stereo|dynamic_range:sdr|encryption:plain|ladder:tv|package:dash|resolution:hd|subs-tag:HotstarVIP|video_codec:vp9', + 'device-id': compat_str(uuid.uuid4()), + 'os-name': 'Windows', + 'os-version': '10', }) class HotStarIE(HotStarBaseIE): IE_NAME = 'hotstar' - _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})' + _VALID_URL = r'https?://(?:www\.)?hotstar\.com/.*(?P<id>\d{10})' _TESTS = [{ # contentData 'url': 'https://www.hotstar.com/can-you-not-spread-rumours/1000076273', @@ -121,7 +147,8 @@ class HotStarIE(HotStarBaseIE): headers = {'Referer': url} formats = [] geo_restricted = False - playback_sets = self._call_api_v2('h/v2/play', video_id)['playBackSets'] + # change to v2 in the future + playback_sets = self._call_api_v2('play/v1/playback', video_id)['playBackSets'] for playback_set in playback_sets: if not isinstance(playback_set, dict): continue diff --git a/youtube_dl/extractor/howcast.py b/youtube_dlc/extractor/howcast.py index 7e36b85ad..7e36b85ad 100644 --- a/youtube_dl/extractor/howcast.py +++ b/youtube_dlc/extractor/howcast.py diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dlc/extractor/howstuffworks.py index cf90ab3c9..cf90ab3c9 100644 --- a/youtube_dl/extractor/howstuffworks.py +++ b/youtube_dlc/extractor/howstuffworks.py diff --git a/youtube_dlc/extractor/hrfensehen.py b/youtube_dlc/extractor/hrfensehen.py new file mode 100644 index 000000000..805345e69 --- /dev/null +++ b/youtube_dlc/extractor/hrfensehen.py @@ -0,0 +1,102 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import json +import re + +from youtube_dlc.utils import int_or_none, unified_timestamp, unescapeHTML +from .common import InfoExtractor + + +class HRFernsehenIE(InfoExtractor): + IE_NAME = 'hrfernsehen' + _VALID_URL = r'^https?://www\.(?:hr-fernsehen|hessenschau)\.de/.*,video-(?P<id>[0-9]{6})\.html' + + _TESTS = [{ + 'url': 'https://www.hessenschau.de/tv-sendung/hessenschau-vom-26082020,video-130546.html', + 'md5': '5c4e0ba94677c516a2f65a84110fc536', + 'info_dict': { + 'id': '130546', + 'ext': 'mp4', + 'description': 'Sturmtief Kirsten fegt über Hessen / Die Corona-Pandemie – eine Chronologie / ' + 'Sterbehilfe: Die Lage in Hessen / Miss Hessen leitet zwei eigene Unternehmen / ' + 'Pop-Up Museum zeigt Schwarze Unterhaltung und Black Music', + 'subtitles': {'de': [{ + 'url': 'https://hr-a.akamaihd.net/video/as/hessenschau/2020_08/hrLogo_200826200407_L385592_512x288-25p-500kbit.vtt' + }]}, + 'timestamp': 1598470200, + 'upload_date': '20200826', + 'thumbnails': [{ + 'url': 'https://www.hessenschau.de/tv-sendung/hs_ganz-1554~_t-1598465545029_v-16to9.jpg', + 'id': '0' + }, { + 'url': 'https://www.hessenschau.de/tv-sendung/hs_ganz-1554~_t-1598465545029_v-16to9__medium.jpg', + 'id': '1' + }], + 'title': 'hessenschau vom 26.08.2020' + } + }, { + 'url': 'https://www.hr-fernsehen.de/sendungen-a-z/mex/sendungen/fair-und-gut---was-hinter-aldis-eigenem-guetesiegel-steckt,video-130544.html', + 'only_matching': True + }] + + _GEO_COUNTRIES = ['DE'] + + def extract_airdate(self, loader_data): + airdate_str = loader_data.get('mediaMetadata', {}).get('agf', {}).get('airdate') + + if airdate_str is None: + return None + + return unified_timestamp(airdate_str) + + def extract_formats(self, loader_data): + stream_formats = [] + for stream_obj in loader_data["videoResolutionLevels"]: + stream_format = { + 'format_id': str(stream_obj['verticalResolution']) + "p", + 'height': stream_obj['verticalResolution'], + 'url': stream_obj['url'], + } + + quality_information = re.search(r'([0-9]{3,4})x([0-9]{3,4})-([0-9]{2})p-([0-9]{3,4})kbit', + stream_obj['url']) + if quality_information: + stream_format['width'] = int_or_none(quality_information.group(1)) + stream_format['height'] = int_or_none(quality_information.group(2)) + stream_format['fps'] = int_or_none(quality_information.group(3)) + stream_format['tbr'] = int_or_none(quality_information.group(4)) + + stream_formats.append(stream_format) + + self._sort_formats(stream_formats) + return stream_formats + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_meta( + ['og:title', 'twitter:title', 'name'], webpage) + description = self._html_search_meta( + ['description'], webpage) + + loader_str = unescapeHTML(self._search_regex(r"data-hr-mediaplayer-loader='([^']*)'", webpage, "ardloader")) + loader_data = json.loads(loader_str) + + info = { + 'id': video_id, + 'title': title, + 'description': description, + 'formats': self.extract_formats(loader_data), + 'timestamp': self.extract_airdate(loader_data) + } + + if "subtitle" in loader_data: + info["subtitles"] = {"de": [{"url": loader_data["subtitle"]}]} + + thumbnails = list(set([t for t in loader_data.get("previewImageUrl", {}).values()])) + if len(thumbnails) > 0: + info["thumbnails"] = [{"url": t} for t in thumbnails] + + return info diff --git a/youtube_dl/extractor/hrti.py b/youtube_dlc/extractor/hrti.py index 23f7b1fc9..23f7b1fc9 100644 --- a/youtube_dl/extractor/hrti.py +++ b/youtube_dlc/extractor/hrti.py diff --git a/youtube_dl/extractor/huajiao.py b/youtube_dlc/extractor/huajiao.py index 4ca275dda..4ca275dda 100644 --- a/youtube_dl/extractor/huajiao.py +++ b/youtube_dlc/extractor/huajiao.py diff --git a/youtube_dl/extractor/huffpost.py b/youtube_dlc/extractor/huffpost.py index 97e36f056..97e36f056 100644 --- a/youtube_dl/extractor/huffpost.py +++ b/youtube_dlc/extractor/huffpost.py diff --git a/youtube_dl/extractor/hungama.py b/youtube_dlc/extractor/hungama.py index 3fdaac5b6..3fdaac5b6 100644 --- a/youtube_dl/extractor/hungama.py +++ b/youtube_dlc/extractor/hungama.py diff --git a/youtube_dl/extractor/hypem.py b/youtube_dlc/extractor/hypem.py index 9ca28d632..9ca28d632 100644 --- a/youtube_dl/extractor/hypem.py +++ b/youtube_dlc/extractor/hypem.py diff --git a/youtube_dl/extractor/ign.py b/youtube_dlc/extractor/ign.py index a96ea8010..a96ea8010 100644 --- a/youtube_dl/extractor/ign.py +++ b/youtube_dlc/extractor/ign.py diff --git a/youtube_dl/extractor/imdb.py b/youtube_dlc/extractor/imdb.py index a31301985..a31301985 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dlc/extractor/imdb.py diff --git a/youtube_dl/extractor/imggaming.py b/youtube_dlc/extractor/imggaming.py index e11f92053..e11f92053 100644 --- a/youtube_dl/extractor/imggaming.py +++ b/youtube_dlc/extractor/imggaming.py diff --git a/youtube_dl/extractor/imgur.py b/youtube_dlc/extractor/imgur.py index a5ba03efa..4dc7b0b5c 100644 --- a/youtube_dl/extractor/imgur.py +++ b/youtube_dlc/extractor/imgur.py @@ -60,7 +60,7 @@ class ImgurIE(InfoExtractor): 'width': width, 'height': height, 'http_headers': { - 'User-Agent': 'youtube-dl (like wget)', + 'User-Agent': 'youtube-dlc (like wget)', }, }) @@ -82,7 +82,7 @@ class ImgurIE(InfoExtractor): 'url': self._proto_relative_url(gifd['gifUrl']), 'filesize': gifd.get('size'), 'http_headers': { - 'User-Agent': 'youtube-dl (like wget)', + 'User-Agent': 'youtube-dlc (like wget)', }, }) diff --git a/youtube_dl/extractor/ina.py b/youtube_dlc/extractor/ina.py index 12695af27..12695af27 100644 --- a/youtube_dl/extractor/ina.py +++ b/youtube_dlc/extractor/ina.py diff --git a/youtube_dl/extractor/inc.py b/youtube_dlc/extractor/inc.py index d5b258a0f..d5b258a0f 100644 --- a/youtube_dl/extractor/inc.py +++ b/youtube_dlc/extractor/inc.py diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dlc/extractor/indavideo.py index 4c16243ec..4c16243ec 100644 --- a/youtube_dl/extractor/indavideo.py +++ b/youtube_dlc/extractor/indavideo.py diff --git a/youtube_dl/extractor/infoq.py b/youtube_dlc/extractor/infoq.py index 18249cf9b..18249cf9b 100644 --- a/youtube_dl/extractor/infoq.py +++ b/youtube_dlc/extractor/infoq.py diff --git a/youtube_dl/extractor/instagram.py b/youtube_dlc/extractor/instagram.py index b061850a1..b061850a1 100644 --- a/youtube_dl/extractor/instagram.py +++ b/youtube_dlc/extractor/instagram.py diff --git a/youtube_dl/extractor/internazionale.py b/youtube_dlc/extractor/internazionale.py index 676e8e269..676e8e269 100644 --- a/youtube_dl/extractor/internazionale.py +++ b/youtube_dlc/extractor/internazionale.py diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dlc/extractor/internetvideoarchive.py index 59b0a90c3..59b0a90c3 100644 --- a/youtube_dl/extractor/internetvideoarchive.py +++ b/youtube_dlc/extractor/internetvideoarchive.py diff --git a/youtube_dl/extractor/iprima.py b/youtube_dlc/extractor/iprima.py index 53a550c11..648ae6741 100644 --- a/youtube_dl/extractor/iprima.py +++ b/youtube_dlc/extractor/iprima.py @@ -86,7 +86,8 @@ class IPrimaIE(InfoExtractor): (r'<iframe[^>]+\bsrc=["\'](?:https?:)?//(?:api\.play-backend\.iprima\.cz/prehravac/embedded|prima\.iprima\.cz/[^/]+/[^/]+)\?.*?\bid=(p\d+)', r'data-product="([^"]+)">', r'id=["\']player-(p\d+)"', - r'playerId\s*:\s*["\']player-(p\d+)'), + r'playerId\s*:\s*["\']player-(p\d+)', + r'\bvideos\s*=\s*["\'](p\d+)'), webpage, 'real id') playerpage = self._download_webpage( diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dlc/extractor/iqiyi.py index cd11aa70f..cd11aa70f 100644 --- a/youtube_dl/extractor/iqiyi.py +++ b/youtube_dlc/extractor/iqiyi.py diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dlc/extractor/ir90tv.py index d5a3f6fa5..d5a3f6fa5 100644 --- a/youtube_dl/extractor/ir90tv.py +++ b/youtube_dlc/extractor/ir90tv.py diff --git a/youtube_dl/extractor/itv.py b/youtube_dlc/extractor/itv.py index ad2f4eca5..ad2f4eca5 100644 --- a/youtube_dl/extractor/itv.py +++ b/youtube_dlc/extractor/itv.py diff --git a/youtube_dl/extractor/ivi.py b/youtube_dlc/extractor/ivi.py index b5a740a01..b9cb5a8e6 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dlc/extractor/ivi.py @@ -142,7 +142,7 @@ class IviIE(InfoExtractor): continue elif bundled: raise ExtractorError( - 'This feature does not work from bundled exe. Run youtube-dl from sources.', + 'This feature does not work from bundled exe. Run youtube-dlc from sources.', expected=True) elif not pycryptodomex_found: raise ExtractorError( diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dlc/extractor/ivideon.py index 3ca824f79..3ca824f79 100644 --- a/youtube_dl/extractor/ivideon.py +++ b/youtube_dlc/extractor/ivideon.py diff --git a/youtube_dl/extractor/iwara.py b/youtube_dlc/extractor/iwara.py index 907d5fc8b..907d5fc8b 100644 --- a/youtube_dl/extractor/iwara.py +++ b/youtube_dlc/extractor/iwara.py diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dlc/extractor/izlesene.py index f8fca6c8f..f8fca6c8f 100644 --- a/youtube_dl/extractor/izlesene.py +++ b/youtube_dlc/extractor/izlesene.py diff --git a/youtube_dl/extractor/jamendo.py b/youtube_dlc/extractor/jamendo.py index 490efa8fb..490efa8fb 100644 --- a/youtube_dl/extractor/jamendo.py +++ b/youtube_dlc/extractor/jamendo.py diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dlc/extractor/jeuxvideo.py index e9f4ed738..e9f4ed738 100644 --- a/youtube_dl/extractor/jeuxvideo.py +++ b/youtube_dlc/extractor/jeuxvideo.py diff --git a/youtube_dl/extractor/joj.py b/youtube_dlc/extractor/joj.py index 62b28e980..637618183 100644 --- a/youtube_dl/extractor/joj.py +++ b/youtube_dlc/extractor/joj.py @@ -1,108 +1,108 @@ -# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
- int_or_none,
- js_to_json,
- try_get,
-)
-
-
-class JojIE(InfoExtractor):
- _VALID_URL = r'''(?x)
- (?:
- joj:|
- https?://media\.joj\.sk/embed/
- )
- (?P<id>[^/?#^]+)
- '''
- _TESTS = [{
- 'url': 'https://media.joj.sk/embed/a388ec4c-6019-4a4a-9312-b1bee194e932',
- 'info_dict': {
- 'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
- 'ext': 'mp4',
- 'title': 'NOVÉ BÝVANIE',
- 'thumbnail': r're:^https?://.*\.jpg$',
- 'duration': 3118,
- }
- }, {
- 'url': 'https://media.joj.sk/embed/9i1cxv',
- 'only_matching': True,
- }, {
- 'url': 'joj:a388ec4c-6019-4a4a-9312-b1bee194e932',
- 'only_matching': True,
- }, {
- 'url': 'joj:9i1cxv',
- 'only_matching': True,
- }]
-
- @staticmethod
- def _extract_urls(webpage):
- return [
- mobj.group('url')
- for mobj in re.finditer(
- r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//media\.joj\.sk/embed/(?:(?!\1).)+)\1',
- webpage)]
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
-
- webpage = self._download_webpage(
- 'https://media.joj.sk/embed/%s' % video_id, video_id)
-
- title = self._search_regex(
- (r'videoTitle\s*:\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
- r'<title>(?P<title>[^<]+)'), webpage, 'title',
- default=None, group='title') or self._og_search_title(webpage)
-
- bitrates = self._parse_json(
- self._search_regex(
- r'(?s)(?:src|bitrates)\s*=\s*({.+?});', webpage, 'bitrates',
- default='{}'),
- video_id, transform_source=js_to_json, fatal=False)
-
- formats = []
- for format_url in try_get(bitrates, lambda x: x['mp4'], list) or []:
- if isinstance(format_url, compat_str):
- height = self._search_regex(
- r'(\d+)[pP]\.', format_url, 'height', default=None)
- formats.append({
- 'url': format_url,
- 'format_id': '%sp' % height if height else None,
- 'height': int(height),
- })
- if not formats:
- playlist = self._download_xml(
- 'https://media.joj.sk/services/Video.php?clip=%s' % video_id,
- video_id)
- for file_el in playlist.findall('./files/file'):
- path = file_el.get('path')
- if not path:
- continue
- format_id = file_el.get('id') or file_el.get('label')
- formats.append({
- 'url': 'http://n16.joj.sk/storage/%s' % path.replace(
- 'dat/', '', 1),
- 'format_id': format_id,
- 'height': int_or_none(self._search_regex(
- r'(\d+)[pP]', format_id or path, 'height',
- default=None)),
- })
- self._sort_formats(formats)
-
- thumbnail = self._og_search_thumbnail(webpage)
-
- duration = int_or_none(self._search_regex(
- r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
-
- return {
- 'id': video_id,
- 'title': title,
- 'thumbnail': thumbnail,
- 'duration': duration,
- 'formats': formats,
- }
+# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import compat_str +from ..utils import ( + int_or_none, + js_to_json, + try_get, +) + + +class JojIE(InfoExtractor): + _VALID_URL = r'''(?x) + (?: + joj:| + https?://media\.joj\.sk/embed/ + ) + (?P<id>[^/?#^]+) + ''' + _TESTS = [{ + 'url': 'https://media.joj.sk/embed/a388ec4c-6019-4a4a-9312-b1bee194e932', + 'info_dict': { + 'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932', + 'ext': 'mp4', + 'title': 'NOVÉ BÝVANIE', + 'thumbnail': r're:^https?://.*\.jpg$', + 'duration': 3118, + } + }, { + 'url': 'https://media.joj.sk/embed/9i1cxv', + 'only_matching': True, + }, { + 'url': 'joj:a388ec4c-6019-4a4a-9312-b1bee194e932', + 'only_matching': True, + }, { + 'url': 'joj:9i1cxv', + 'only_matching': True, + }] + + @staticmethod + def _extract_urls(webpage): + return [ + mobj.group('url') + for mobj in re.finditer( + r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//media\.joj\.sk/embed/(?:(?!\1).)+)\1', + webpage)] + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage( + 'https://media.joj.sk/embed/%s' % video_id, video_id) + + title = self._search_regex( + (r'videoTitle\s*:\s*(["\'])(?P<title>(?:(?!\1).)+)\1', + r'<title>(?P<title>[^<]+)'), webpage, 'title', + default=None, group='title') or self._og_search_title(webpage) + + bitrates = self._parse_json( + self._search_regex( + r'(?s)(?:src|bitrates)\s*=\s*({.+?});', webpage, 'bitrates', + default='{}'), + video_id, transform_source=js_to_json, fatal=False) + + formats = [] + for format_url in try_get(bitrates, lambda x: x['mp4'], list) or []: + if isinstance(format_url, compat_str): + height = self._search_regex( + r'(\d+)[pP]\.', format_url, 'height', default=None) + formats.append({ + 'url': format_url, + 'format_id': '%sp' % height if height else None, + 'height': int(height), + }) + if not formats: + playlist = self._download_xml( + 'https://media.joj.sk/services/Video.php?clip=%s' % video_id, + video_id) + for file_el in playlist.findall('./files/file'): + path = file_el.get('path') + if not path: + continue + format_id = file_el.get('id') or file_el.get('label') + formats.append({ + 'url': 'http://n16.joj.sk/storage/%s' % path.replace( + 'dat/', '', 1), + 'format_id': format_id, + 'height': int_or_none(self._search_regex( + r'(\d+)[pP]', format_id or path, 'height', + default=None)), + }) + self._sort_formats(formats) + + thumbnail = self._og_search_thumbnail(webpage) + + duration = int_or_none(self._search_regex( + r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False)) + + return { + 'id': video_id, + 'title': title, + 'thumbnail': thumbnail, + 'duration': duration, + 'formats': formats, + } diff --git a/youtube_dl/extractor/jove.py b/youtube_dlc/extractor/jove.py index 27e0e37f6..27e0e37f6 100644 --- a/youtube_dl/extractor/jove.py +++ b/youtube_dlc/extractor/jove.py diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dlc/extractor/jwplatform.py index c34b5f5e6..c34b5f5e6 100644 --- a/youtube_dl/extractor/jwplatform.py +++ b/youtube_dlc/extractor/jwplatform.py diff --git a/youtube_dl/extractor/kakao.py b/youtube_dlc/extractor/kakao.py index 32935bb28..fefd8a215 100644 --- a/youtube_dl/extractor/kakao.py +++ b/youtube_dlc/extractor/kakao.py @@ -8,13 +8,13 @@ from ..utils import ( int_or_none, strip_or_none, unified_timestamp, - update_url_query, ) class KakaoIE(InfoExtractor): _VALID_URL = r'https?://(?:play-)?tv\.kakao\.com/(?:channel/\d+|embed/player)/cliplink/(?P<id>\d+|[^?#&]+@my)' - _API_BASE_TMPL = 'http://tv.kakao.com/api/v1/ft/cliplinks/%s/' + _API_BASE_TMPL = 'http://tv.kakao.com/api/v1/ft/playmeta/cliplink/%s/' + _CDN_API = 'https://tv.kakao.com/katz/v1/ft/cliplink/%s/readyNplay?' _TESTS = [{ 'url': 'http://tv.kakao.com/channel/2671005/cliplink/301965083', @@ -45,18 +45,8 @@ class KakaoIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - display_id = video_id.rstrip('@my') api_base = self._API_BASE_TMPL % video_id - - player_header = { - 'Referer': update_url_query( - 'http://tv.kakao.com/embed/player/cliplink/%s' % video_id, { - 'service': 'kakao_tv', - 'autoplay': '1', - 'profile': 'HIGH', - 'wmode': 'transparent', - }) - } + cdn_api_base = self._CDN_API % video_id query = { 'player': 'monet_html5', @@ -73,17 +63,14 @@ class KakaoIE(InfoExtractor): 'videoOutputList', 'width', 'height', 'kbps', 'profile', 'label']) } - impress = self._download_json( - api_base + 'impress', display_id, 'Downloading video info', - query=query, headers=player_header) + api_json = self._download_json( + api_base, video_id, 'Downloading video info') - clip_link = impress['clipLink'] + clip_link = api_json['clipLink'] clip = clip_link['clip'] title = clip.get('title') or clip_link.get('displayTitle') - query['tid'] = impress.get('tid', '') - formats = [] for fmt in clip.get('videoOutputList', []): try: @@ -94,15 +81,17 @@ class KakaoIE(InfoExtractor): 'profile': profile_name, 'fields': '-*,url', }) + fmt_url_json = self._download_json( - api_base + 'raw/videolocation', display_id, + cdn_api_base, video_id, 'Downloading video URL for profile %s' % profile_name, - query=query, headers=player_header, fatal=False) + query=query, fatal=False) if fmt_url_json is None: continue - fmt_url = fmt_url_json['url'] + fmt_vidLocation = fmt_url_json['videoLocation'] + fmt_url = fmt_vidLocation['url'] formats.append({ 'url': fmt_url, 'format_id': profile_name, @@ -131,7 +120,7 @@ class KakaoIE(InfoExtractor): }) return { - 'id': display_id, + 'id': video_id, 'title': title, 'description': strip_or_none(clip.get('description')), 'uploader': clip_link.get('channel', {}).get('name'), diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dlc/extractor/kaltura.py index 49d13460d..49d13460d 100644 --- a/youtube_dl/extractor/kaltura.py +++ b/youtube_dlc/extractor/kaltura.py diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dlc/extractor/kanalplay.py index 6c3498c67..6c3498c67 100644 --- a/youtube_dl/extractor/kanalplay.py +++ b/youtube_dlc/extractor/kanalplay.py diff --git a/youtube_dl/extractor/kankan.py b/youtube_dlc/extractor/kankan.py index a677ff447..a677ff447 100644 --- a/youtube_dl/extractor/kankan.py +++ b/youtube_dlc/extractor/kankan.py diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dlc/extractor/karaoketv.py index bfccf89b0..bfccf89b0 100644 --- a/youtube_dl/extractor/karaoketv.py +++ b/youtube_dlc/extractor/karaoketv.py diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dlc/extractor/karrierevideos.py index 7b291e0a0..7b291e0a0 100644 --- a/youtube_dl/extractor/karrierevideos.py +++ b/youtube_dlc/extractor/karrierevideos.py diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dlc/extractor/keezmovies.py index c3eb74c17..c3eb74c17 100644 --- a/youtube_dl/extractor/keezmovies.py +++ b/youtube_dlc/extractor/keezmovies.py diff --git a/youtube_dl/extractor/ketnet.py b/youtube_dlc/extractor/ketnet.py index 93a98e1e0..93a98e1e0 100644 --- a/youtube_dl/extractor/ketnet.py +++ b/youtube_dlc/extractor/ketnet.py diff --git a/youtube_dl/extractor/khanacademy.py b/youtube_dlc/extractor/khanacademy.py index 61739efa7..61739efa7 100644 --- a/youtube_dl/extractor/khanacademy.py +++ b/youtube_dlc/extractor/khanacademy.py diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dlc/extractor/kickstarter.py index d4da8f484..d4da8f484 100644 --- a/youtube_dl/extractor/kickstarter.py +++ b/youtube_dlc/extractor/kickstarter.py diff --git a/youtube_dl/extractor/kinja.py b/youtube_dlc/extractor/kinja.py index 79e3026d2..79e3026d2 100644 --- a/youtube_dl/extractor/kinja.py +++ b/youtube_dlc/extractor/kinja.py diff --git a/youtube_dl/extractor/kinopoisk.py b/youtube_dlc/extractor/kinopoisk.py index 9e8d01f53..9e8d01f53 100644 --- a/youtube_dl/extractor/kinopoisk.py +++ b/youtube_dlc/extractor/kinopoisk.py diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dlc/extractor/konserthusetplay.py index dd42bb2f2..dd42bb2f2 100644 --- a/youtube_dl/extractor/konserthusetplay.py +++ b/youtube_dlc/extractor/konserthusetplay.py diff --git a/youtube_dl/extractor/krasview.py b/youtube_dlc/extractor/krasview.py index d27d052ff..d27d052ff 100644 --- a/youtube_dl/extractor/krasview.py +++ b/youtube_dlc/extractor/krasview.py diff --git a/youtube_dl/extractor/ku6.py b/youtube_dlc/extractor/ku6.py index a574408e5..a574408e5 100644 --- a/youtube_dl/extractor/ku6.py +++ b/youtube_dlc/extractor/ku6.py diff --git a/youtube_dl/extractor/kusi.py b/youtube_dlc/extractor/kusi.py index 6a7e3baa7..6a7e3baa7 100644 --- a/youtube_dl/extractor/kusi.py +++ b/youtube_dlc/extractor/kusi.py diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dlc/extractor/kuwo.py index cc5b2a1c1..cc5b2a1c1 100644 --- a/youtube_dl/extractor/kuwo.py +++ b/youtube_dlc/extractor/kuwo.py diff --git a/youtube_dl/extractor/la7.py b/youtube_dlc/extractor/la7.py index c3b4ffa7e..c3b4ffa7e 100644 --- a/youtube_dl/extractor/la7.py +++ b/youtube_dlc/extractor/la7.py diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dlc/extractor/laola1tv.py index fa217365a..fa217365a 100644 --- a/youtube_dl/extractor/laola1tv.py +++ b/youtube_dlc/extractor/laola1tv.py diff --git a/youtube_dl/extractor/lci.py b/youtube_dlc/extractor/lci.py index 920872f5c..920872f5c 100644 --- a/youtube_dl/extractor/lci.py +++ b/youtube_dlc/extractor/lci.py diff --git a/youtube_dl/extractor/lcp.py b/youtube_dlc/extractor/lcp.py index ade27a99e..ade27a99e 100644 --- a/youtube_dl/extractor/lcp.py +++ b/youtube_dlc/extractor/lcp.py diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dlc/extractor/lecture2go.py index 81b5d41be..81b5d41be 100644 --- a/youtube_dl/extractor/lecture2go.py +++ b/youtube_dlc/extractor/lecture2go.py diff --git a/youtube_dl/extractor/lecturio.py b/youtube_dlc/extractor/lecturio.py index 1b2dcef46..1b2dcef46 100644 --- a/youtube_dl/extractor/lecturio.py +++ b/youtube_dlc/extractor/lecturio.py diff --git a/youtube_dl/extractor/leeco.py b/youtube_dlc/extractor/leeco.py index 7dc0ad794..7dc0ad794 100644 --- a/youtube_dl/extractor/leeco.py +++ b/youtube_dlc/extractor/leeco.py diff --git a/youtube_dl/extractor/lego.py b/youtube_dlc/extractor/lego.py index 1e3c19dfd..1e3c19dfd 100644 --- a/youtube_dl/extractor/lego.py +++ b/youtube_dlc/extractor/lego.py diff --git a/youtube_dl/extractor/lemonde.py b/youtube_dlc/extractor/lemonde.py index 3306892e8..3306892e8 100644 --- a/youtube_dl/extractor/lemonde.py +++ b/youtube_dlc/extractor/lemonde.py diff --git a/youtube_dl/extractor/lenta.py b/youtube_dlc/extractor/lenta.py index 2ebd4e577..2ebd4e577 100644 --- a/youtube_dl/extractor/lenta.py +++ b/youtube_dlc/extractor/lenta.py diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dlc/extractor/libraryofcongress.py index 03f205144..03f205144 100644 --- a/youtube_dl/extractor/libraryofcongress.py +++ b/youtube_dlc/extractor/libraryofcongress.py diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dlc/extractor/libsyn.py index 2cf444258..2cf444258 100644 --- a/youtube_dl/extractor/libsyn.py +++ b/youtube_dlc/extractor/libsyn.py diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dlc/extractor/lifenews.py index 42e263bfa..42e263bfa 100644 --- a/youtube_dl/extractor/lifenews.py +++ b/youtube_dlc/extractor/lifenews.py diff --git a/youtube_dl/extractor/limelight.py b/youtube_dlc/extractor/limelight.py index 39f74d282..39f74d282 100644 --- a/youtube_dl/extractor/limelight.py +++ b/youtube_dlc/extractor/limelight.py diff --git a/youtube_dl/extractor/line.py b/youtube_dlc/extractor/line.py index 7f5fa446e..7f5fa446e 100644 --- a/youtube_dl/extractor/line.py +++ b/youtube_dlc/extractor/line.py diff --git a/youtube_dl/extractor/linkedin.py b/youtube_dlc/extractor/linkedin.py index 26fc703d1..26fc703d1 100644 --- a/youtube_dl/extractor/linkedin.py +++ b/youtube_dlc/extractor/linkedin.py diff --git a/youtube_dl/extractor/linuxacademy.py b/youtube_dlc/extractor/linuxacademy.py index 23ca965d9..23ca965d9 100644 --- a/youtube_dl/extractor/linuxacademy.py +++ b/youtube_dlc/extractor/linuxacademy.py diff --git a/youtube_dl/extractor/litv.py b/youtube_dlc/extractor/litv.py index 337b1b15c..337b1b15c 100644 --- a/youtube_dl/extractor/litv.py +++ b/youtube_dlc/extractor/litv.py diff --git a/youtube_dl/extractor/livejournal.py b/youtube_dlc/extractor/livejournal.py index 3a9f4553f..3a9f4553f 100644 --- a/youtube_dl/extractor/livejournal.py +++ b/youtube_dlc/extractor/livejournal.py diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dlc/extractor/liveleak.py index 4ac437c8b..4ac437c8b 100644 --- a/youtube_dl/extractor/liveleak.py +++ b/youtube_dlc/extractor/liveleak.py diff --git a/youtube_dl/extractor/livestream.py b/youtube_dlc/extractor/livestream.py index e55b1a202..e55b1a202 100644 --- a/youtube_dl/extractor/livestream.py +++ b/youtube_dlc/extractor/livestream.py diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dlc/extractor/lnkgo.py index 3e71852aa..3e71852aa 100644 --- a/youtube_dl/extractor/lnkgo.py +++ b/youtube_dlc/extractor/lnkgo.py diff --git a/youtube_dl/extractor/localnews8.py b/youtube_dlc/extractor/localnews8.py index aad396135..aad396135 100644 --- a/youtube_dl/extractor/localnews8.py +++ b/youtube_dlc/extractor/localnews8.py diff --git a/youtube_dl/extractor/lovehomeporn.py b/youtube_dlc/extractor/lovehomeporn.py index 8f65a3c03..8f65a3c03 100644 --- a/youtube_dl/extractor/lovehomeporn.py +++ b/youtube_dlc/extractor/lovehomeporn.py diff --git a/youtube_dl/extractor/lrt.py b/youtube_dlc/extractor/lrt.py index f5c997ef4..f5c997ef4 100644 --- a/youtube_dl/extractor/lrt.py +++ b/youtube_dlc/extractor/lrt.py diff --git a/youtube_dl/extractor/lynda.py b/youtube_dlc/extractor/lynda.py index b3d8653d0..b3d8653d0 100644 --- a/youtube_dl/extractor/lynda.py +++ b/youtube_dlc/extractor/lynda.py diff --git a/youtube_dl/extractor/m6.py b/youtube_dlc/extractor/m6.py index 9806875e8..9806875e8 100644 --- a/youtube_dl/extractor/m6.py +++ b/youtube_dlc/extractor/m6.py diff --git a/youtube_dlc/extractor/magentamusik360.py b/youtube_dlc/extractor/magentamusik360.py new file mode 100644 index 000000000..5c274902f --- /dev/null +++ b/youtube_dlc/extractor/magentamusik360.py @@ -0,0 +1,61 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class MagentaMusik360IE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?magenta-musik-360\.de/([a-z0-9-]+-(?P<id>[0-9]+)|festivals/.+)' + _TESTS = [{ + 'url': 'https://www.magenta-musik-360.de/within-temptation-wacken-2019-1-9208205928595185932', + 'md5': '65b6f060b40d90276ec6fb9b992c1216', + 'info_dict': { + 'id': '9208205928595185932', + 'ext': 'm3u8', + 'title': 'WITHIN TEMPTATION', + 'description': 'Robert Westerholt und Sharon Janny den Adel gründeten die Symphonic Metal-Band. Privat sind die Niederländer ein Paar und haben zwei Kinder. Die Single Ice Queen brachte ihnen Platin und Gold und verhalf 2002 zum internationalen Durchbruch. Charakteristisch für die Band war Anfangs der hohe Gesang von Frontfrau Sharon. Stilistisch fing die Band im Gothic Metal an. Mit neuem Sound, schnellen Gitarrenriffs und Gitarrensoli, avancierte Within Temptation zur erfolgreichen Rockband. Auch dieses Jahr wird die Band ihre Fangemeinde wieder mitreißen.', + } + }, { + 'url': 'https://www.magenta-musik-360.de/festivals/wacken-world-wide-2020-body-count-feat-ice-t', + 'md5': '81010d27d7cab3f7da0b0f681b983b7e', + 'info_dict': { + 'id': '9208205928595231363', + 'ext': 'm3u8', + 'title': 'Body Count feat. Ice-T', + 'description': 'Body Count feat. Ice-T konnten bereits im vergangenen Jahr auf dem „Holy Ground“ in Wacken überzeugen. 2020 gehen die Crossover-Metaller aus einem Club in Los Angeles auf Sendung und bringen mit ihrer Mischung aus Metal und Hip-Hop Abwechslung und ordentlich Alarm zum WWW. Bereits seit 1990 stehen die beiden Gründer Ice-T (Gesang) und Ernie C (Gitarre) auf der Bühne. Sieben Studioalben hat die Gruppe bis jetzt veröffentlicht, darunter das Debüt „Body Count“ (1992) mit dem kontroversen Track „Cop Killer“.', + } + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + # _match_id casts to string, but since "None" is not a valid video_id for magenta + # there is no risk for confusion + if video_id == "None": + webpage = self._download_webpage(url, video_id) + video_id = self._html_search_regex(r'data-asset-id="([^"]+)"', webpage, 'video_id') + json = self._download_json("https://wcps.t-online.de/cvss/magentamusic/vodplayer/v3/player/58935/%s/Main%%20Movie" % video_id, video_id) + xml_url = json['content']['feature']['representations'][0]['contentPackages'][0]['media']['href'] + metadata = json['content']['feature'].get('metadata') + title = None + description = None + duration = None + thumbnails = [] + if metadata: + title = metadata.get('title') + description = metadata.get('fullDescription') + duration = metadata.get('runtimeInSeconds') + for img_key in ('teaserImageWide', 'smallCoverImage'): + if img_key in metadata: + thumbnails.append({'url': metadata[img_key].get('href')}) + + xml = self._download_xml(xml_url, video_id) + final_url = xml[0][0][0].attrib['src'] + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'url': final_url, + 'duration': duration, + 'thumbnails': thumbnails + } diff --git a/youtube_dl/extractor/mailru.py b/youtube_dlc/extractor/mailru.py index 65cc474db..6fdf70aa6 100644 --- a/youtube_dl/extractor/mailru.py +++ b/youtube_dlc/extractor/mailru.py @@ -20,10 +20,10 @@ class MailRuIE(InfoExtractor): IE_DESC = 'Видео@Mail.Ru' _VALID_URL = r'''(?x) https?:// - (?:(?:www|m)\.)?my\.mail\.ru/+ + (?:(?:www|m|videoapi)\.)?my\.mail\.ru/+ (?: video/.*\#video=/?(?P<idv1>(?:[^/]+/){3}\d+)| - (?:(?P<idv2prefix>(?:[^/]+/+){2})video/(?P<idv2suffix>[^/]+/\d+))\.html| + (?:videos/embed/)?(?:(?P<idv2prefix>(?:[^/]+/+){2})(?:video/(?:embed/)?)?(?P<idv2suffix>[^/]+/\d+))(?:\.html)?| (?:video/embed|\+/video/meta)/(?P<metaid>\d+) ) ''' @@ -108,15 +108,21 @@ class MailRuIE(InfoExtractor): if not video_id: video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix') webpage = self._download_webpage(url, video_id) - page_config = self._parse_json(self._search_regex( + page_config = self._parse_json(self._search_regex([ r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>', + r'(?s)"video":\s*(\{.+?\}),'], webpage, 'page config', default='{}'), video_id, fatal=False) if page_config: - meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl') + meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl') or page_config.get('metadataUrl') else: meta_url = None video_data = None + + # fix meta_url if missing the host address + if re.match(r'^\/\+\/', meta_url): + meta_url = 'https://my.mail.ru' + meta_url + if meta_url: video_data = self._download_json( meta_url, video_id or meta_id, 'Downloading video meta JSON', diff --git a/youtube_dl/extractor/malltv.py b/youtube_dlc/extractor/malltv.py index 6f4fd927f..6f4fd927f 100644 --- a/youtube_dl/extractor/malltv.py +++ b/youtube_dlc/extractor/malltv.py diff --git a/youtube_dl/extractor/mangomolo.py b/youtube_dlc/extractor/mangomolo.py index acee370e9..acee370e9 100644 --- a/youtube_dl/extractor/mangomolo.py +++ b/youtube_dlc/extractor/mangomolo.py diff --git a/youtube_dl/extractor/manyvids.py b/youtube_dlc/extractor/manyvids.py index e8d7163e4..e8d7163e4 100644 --- a/youtube_dl/extractor/manyvids.py +++ b/youtube_dlc/extractor/manyvids.py diff --git a/youtube_dl/extractor/markiza.py b/youtube_dlc/extractor/markiza.py index def960a0c..def960a0c 100644 --- a/youtube_dl/extractor/markiza.py +++ b/youtube_dlc/extractor/markiza.py diff --git a/youtube_dl/extractor/massengeschmacktv.py b/youtube_dlc/extractor/massengeschmacktv.py index cfcc6b224..cfcc6b224 100644 --- a/youtube_dl/extractor/massengeschmacktv.py +++ b/youtube_dlc/extractor/massengeschmacktv.py diff --git a/youtube_dl/extractor/matchtv.py b/youtube_dlc/extractor/matchtv.py index bc9933a81..bc9933a81 100644 --- a/youtube_dl/extractor/matchtv.py +++ b/youtube_dlc/extractor/matchtv.py diff --git a/youtube_dl/extractor/mdr.py b/youtube_dlc/extractor/mdr.py index 322e5b45a..322e5b45a 100644 --- a/youtube_dl/extractor/mdr.py +++ b/youtube_dlc/extractor/mdr.py diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dlc/extractor/medialaan.py index 50d5db802..50d5db802 100644 --- a/youtube_dl/extractor/medialaan.py +++ b/youtube_dlc/extractor/medialaan.py diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dlc/extractor/mediaset.py index 933df1495..933df1495 100644 --- a/youtube_dl/extractor/mediaset.py +++ b/youtube_dlc/extractor/mediaset.py diff --git a/youtube_dl/extractor/mediasite.py b/youtube_dlc/extractor/mediasite.py index d6eb15740..d6eb15740 100644 --- a/youtube_dl/extractor/mediasite.py +++ b/youtube_dlc/extractor/mediasite.py diff --git a/youtube_dl/extractor/medici.py b/youtube_dlc/extractor/medici.py index cd910238e..cd910238e 100644 --- a/youtube_dl/extractor/medici.py +++ b/youtube_dlc/extractor/medici.py diff --git a/youtube_dl/extractor/megaphone.py b/youtube_dlc/extractor/megaphone.py index 5bafa6cf4..5bafa6cf4 100644 --- a/youtube_dl/extractor/megaphone.py +++ b/youtube_dlc/extractor/megaphone.py diff --git a/youtube_dl/extractor/meipai.py b/youtube_dlc/extractor/meipai.py index 2445b8b39..2445b8b39 100644 --- a/youtube_dl/extractor/meipai.py +++ b/youtube_dlc/extractor/meipai.py diff --git a/youtube_dl/extractor/melonvod.py b/youtube_dlc/extractor/melonvod.py index bd8cf13ab..bd8cf13ab 100644 --- a/youtube_dl/extractor/melonvod.py +++ b/youtube_dlc/extractor/melonvod.py diff --git a/youtube_dl/extractor/meta.py b/youtube_dlc/extractor/meta.py index cdb46e163..cdb46e163 100644 --- a/youtube_dl/extractor/meta.py +++ b/youtube_dlc/extractor/meta.py diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dlc/extractor/metacafe.py index 9e92416d1..9e92416d1 100644 --- a/youtube_dl/extractor/metacafe.py +++ b/youtube_dlc/extractor/metacafe.py diff --git a/youtube_dl/extractor/metacritic.py b/youtube_dlc/extractor/metacritic.py index 7d468d78b..7d468d78b 100644 --- a/youtube_dl/extractor/metacritic.py +++ b/youtube_dlc/extractor/metacritic.py diff --git a/youtube_dl/extractor/mgoon.py b/youtube_dlc/extractor/mgoon.py index 7bb473900..7bb473900 100644 --- a/youtube_dl/extractor/mgoon.py +++ b/youtube_dlc/extractor/mgoon.py diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dlc/extractor/mgtv.py index 71fc3ec56..71fc3ec56 100644 --- a/youtube_dl/extractor/mgtv.py +++ b/youtube_dlc/extractor/mgtv.py diff --git a/youtube_dl/extractor/miaopai.py b/youtube_dlc/extractor/miaopai.py index f9e35ac7f..f9e35ac7f 100644 --- a/youtube_dl/extractor/miaopai.py +++ b/youtube_dlc/extractor/miaopai.py diff --git a/youtube_dl/extractor/microsoftvirtualacademy.py b/youtube_dlc/extractor/microsoftvirtualacademy.py index 8e0aee0e6..8e0aee0e6 100644 --- a/youtube_dl/extractor/microsoftvirtualacademy.py +++ b/youtube_dlc/extractor/microsoftvirtualacademy.py diff --git a/youtube_dl/extractor/ministrygrid.py b/youtube_dlc/extractor/ministrygrid.py index 8ad9239c5..8ad9239c5 100644 --- a/youtube_dl/extractor/ministrygrid.py +++ b/youtube_dlc/extractor/ministrygrid.py diff --git a/youtube_dl/extractor/minoto.py b/youtube_dlc/extractor/minoto.py index 636731195..636731195 100644 --- a/youtube_dl/extractor/minoto.py +++ b/youtube_dlc/extractor/minoto.py diff --git a/youtube_dl/extractor/miomio.py b/youtube_dlc/extractor/miomio.py index 40f72d66f..40f72d66f 100644 --- a/youtube_dl/extractor/miomio.py +++ b/youtube_dlc/extractor/miomio.py diff --git a/youtube_dl/extractor/mit.py b/youtube_dlc/extractor/mit.py index e1506a745..e1506a745 100644 --- a/youtube_dl/extractor/mit.py +++ b/youtube_dlc/extractor/mit.py diff --git a/youtube_dl/extractor/mitele.py b/youtube_dlc/extractor/mitele.py index ad9da9612..7f5718e21 100644 --- a/youtube_dl/extractor/mitele.py +++ b/youtube_dlc/extractor/mitele.py @@ -1,5 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals +import json from .common import InfoExtractor from ..utils import ( @@ -31,7 +32,6 @@ class MiTeleIE(InfoExtractor): 'timestamp': 1471209401, 'upload_date': '20160814', }, - 'add_ie': ['Ooyala'], }, { # no explicit title 'url': 'http://www.mitele.es/programas-tv/cuarto-milenio/57b0de3dc915da14058b4876/player', @@ -53,8 +53,7 @@ class MiTeleIE(InfoExtractor): }, 'params': { 'skip_download': True, - }, - 'add_ie': ['Ooyala'], + } }, { 'url': 'http://www.mitele.es/series-online/la-que-se-avecina/57aac5c1c915da951a8b45ed/player', 'only_matching': True, @@ -75,10 +74,7 @@ class MiTeleIE(InfoExtractor): content = pre_player.get('content') or {} info = content.get('info') or {} - return { - '_type': 'url_transparent', - # for some reason only HLS is supported - 'url': smuggle_url('ooyala:' + video_id, {'supportedformats': 'm3u8,dash'}), + info = { 'id': video_id, 'title': title, 'description': info.get('synopsis'), @@ -91,3 +87,34 @@ class MiTeleIE(InfoExtractor): 'age_limit': int_or_none(info.get('rating')), 'timestamp': parse_iso8601(pre_player.get('publishedTime')), } + + if video.get('dataCmsId') == 'ooyala': + info.update({ + '_type': 'url_transparent', + # for some reason only HLS is supported + 'url': smuggle_url('ooyala:' + video_id, {'supportedformats': 'm3u8,dash'}), + }) + else: + config = self._download_json( + video['dataConfig'], video_id, 'Downloading config JSON') + services = config['services'] + gbx = self._download_json( + services['gbx'], video_id, 'Downloading gbx JSON') + caronte = self._download_json( + services['caronte'], video_id, 'Downloading caronte JSON') + cerbero = self._download_json( + caronte['cerbero'], video_id, 'Downloading cerbero JSON', + headers={ + 'Content-Type': 'application/json;charset=UTF-8', + 'Origin': 'https://www.mitele.es' + }, + data=json.dumps({ + 'bbx': caronte['bbx'], + 'gbx': gbx['gbx'] + }).encode('utf-8')) + formats = self._extract_m3u8_formats( + caronte['dls'][0]['stream'], video_id, 'mp4', 'm3u8_native', m3u8_id='hls', + query=dict([cerbero['tokens']['1']['cdn'].split('=', 1)])) + info['formats'] = formats + + return info diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dlc/extractor/mixcloud.py index 9759560f1..9759560f1 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dlc/extractor/mixcloud.py diff --git a/youtube_dl/extractor/mlb.py b/youtube_dlc/extractor/mlb.py index b907f6b49..b907f6b49 100644 --- a/youtube_dl/extractor/mlb.py +++ b/youtube_dlc/extractor/mlb.py diff --git a/youtube_dl/extractor/mnet.py b/youtube_dlc/extractor/mnet.py index 0e26ca1b3..0e26ca1b3 100644 --- a/youtube_dl/extractor/mnet.py +++ b/youtube_dlc/extractor/mnet.py diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dlc/extractor/moevideo.py index eb9b4ce7c..eb9b4ce7c 100644 --- a/youtube_dl/extractor/moevideo.py +++ b/youtube_dlc/extractor/moevideo.py diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dlc/extractor/mofosex.py index 5234cac02..5234cac02 100644 --- a/youtube_dl/extractor/mofosex.py +++ b/youtube_dlc/extractor/mofosex.py diff --git a/youtube_dl/extractor/mojvideo.py b/youtube_dlc/extractor/mojvideo.py index 165e658c9..165e658c9 100644 --- a/youtube_dl/extractor/mojvideo.py +++ b/youtube_dlc/extractor/mojvideo.py diff --git a/youtube_dl/extractor/morningstar.py b/youtube_dlc/extractor/morningstar.py index 0093bcd6c..0093bcd6c 100644 --- a/youtube_dl/extractor/morningstar.py +++ b/youtube_dlc/extractor/morningstar.py diff --git a/youtube_dl/extractor/motherless.py b/youtube_dlc/extractor/motherless.py index b1615b4d8..b1615b4d8 100644 --- a/youtube_dl/extractor/motherless.py +++ b/youtube_dlc/extractor/motherless.py diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dlc/extractor/motorsport.py index c9d1ab64d..c9d1ab64d 100644 --- a/youtube_dl/extractor/motorsport.py +++ b/youtube_dlc/extractor/motorsport.py diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dlc/extractor/movieclips.py index 5453da1ac..5453da1ac 100644 --- a/youtube_dl/extractor/movieclips.py +++ b/youtube_dlc/extractor/movieclips.py diff --git a/youtube_dl/extractor/moviezine.py b/youtube_dlc/extractor/moviezine.py index 85cc6e22f..85cc6e22f 100644 --- a/youtube_dl/extractor/moviezine.py +++ b/youtube_dlc/extractor/moviezine.py diff --git a/youtube_dl/extractor/movingimage.py b/youtube_dlc/extractor/movingimage.py index 4f62d628a..4f62d628a 100644 --- a/youtube_dl/extractor/movingimage.py +++ b/youtube_dlc/extractor/movingimage.py diff --git a/youtube_dl/extractor/msn.py b/youtube_dlc/extractor/msn.py index e59b0b7b0..e59b0b7b0 100644 --- a/youtube_dl/extractor/msn.py +++ b/youtube_dlc/extractor/msn.py diff --git a/youtube_dl/extractor/mtv.py b/youtube_dlc/extractor/mtv.py index fedd5f46b..e545a9ef3 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dlc/extractor/mtv.py @@ -7,6 +7,7 @@ from .common import InfoExtractor from ..compat import ( compat_str, compat_xpath, + compat_urlparse, ) from ..utils import ( ExtractorError, @@ -22,6 +23,7 @@ from ..utils import ( unescapeHTML, update_url_query, url_basename, + get_domain, xpath_text, ) @@ -253,7 +255,42 @@ class MTVServicesInfoExtractor(InfoExtractor): return try_get(feed, lambda x: x['result']['data']['id'], compat_str) - def _extract_mgid(self, webpage): + def _extract_new_triforce_mgid(self, webpage, url='', video_id=None): + # print(compat_urlparse.urlparse(url).netloc) + if url == '': + return + domain = get_domain(url) + if domain is None: + raise ExtractorError( + '[%s] could not get domain' % self.IE_NAME, + expected=True) + url = url.replace("https://", "http://") + enc_url = compat_urlparse.quote(url, safe='') + _TRIFORCE_V8_TEMPLATE = 'https://%s/feeds/triforce/manifest/v8?url=%s' + triforce_manifest_url = _TRIFORCE_V8_TEMPLATE % (domain, enc_url) + + manifest = self._download_json(triforce_manifest_url, video_id, fatal=False) + if manifest: + if manifest.get('manifest').get('type') == 'redirect': + self.to_screen('Found a redirect. Downloading manifest from new location') + new_loc = manifest.get('manifest').get('newLocation') + new_loc = new_loc.replace("https://", "http://") + enc_new_loc = compat_urlparse.quote(new_loc, safe='') + triforce_manifest_new_loc = _TRIFORCE_V8_TEMPLATE % (domain, enc_new_loc) + manifest = self._download_json(triforce_manifest_new_loc, video_id, fatal=False) + + item_id = try_get(manifest, lambda x: x['manifest']['reporting']['itemId'], compat_str) + if not item_id: + self.to_screen('Found no id!') + return + + # 'episode' can be anything. 'content' is used often as well + _MGID_TEMPLATE = 'mgid:arc:episode:%s:%s' + mgid = _MGID_TEMPLATE % (domain, item_id) + + return mgid + + def _extract_mgid(self, webpage, url, data_zone=None): try: # the url can be http://media.mtvnservices.com/fb/{mgid}.swf # or http://media.mtvnservices.com/{mgid} @@ -276,14 +313,17 @@ class MTVServicesInfoExtractor(InfoExtractor): r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid', default=None) if not mgid: - mgid = self._extract_triforce_mgid(webpage) + mgid = self._extract_new_triforce_mgid(webpage, url) + + if not mgid: + mgid = self._extract_triforce_mgid(webpage, data_zone) return mgid def _real_extract(self, url): title = url_basename(url) webpage = self._download_webpage(url, title) - mgid = self._extract_mgid(webpage) + mgid = self._extract_mgid(webpage, url) videos_info = self._get_videos_info(mgid) return videos_info diff --git a/youtube_dl/extractor/muenchentv.py b/youtube_dlc/extractor/muenchentv.py index 2cc2bf229..2cc2bf229 100644 --- a/youtube_dl/extractor/muenchentv.py +++ b/youtube_dlc/extractor/muenchentv.py diff --git a/youtube_dl/extractor/mwave.py b/youtube_dlc/extractor/mwave.py index a67276596..a67276596 100644 --- a/youtube_dl/extractor/mwave.py +++ b/youtube_dlc/extractor/mwave.py diff --git a/youtube_dl/extractor/mychannels.py b/youtube_dlc/extractor/mychannels.py index b1ffe7848..b1ffe7848 100644 --- a/youtube_dl/extractor/mychannels.py +++ b/youtube_dlc/extractor/mychannels.py diff --git a/youtube_dl/extractor/myspace.py b/youtube_dlc/extractor/myspace.py index e164d5940..e164d5940 100644 --- a/youtube_dl/extractor/myspace.py +++ b/youtube_dlc/extractor/myspace.py diff --git a/youtube_dl/extractor/myspass.py b/youtube_dlc/extractor/myspass.py index db7ebc94c..db7ebc94c 100644 --- a/youtube_dl/extractor/myspass.py +++ b/youtube_dlc/extractor/myspass.py diff --git a/youtube_dl/extractor/myvi.py b/youtube_dlc/extractor/myvi.py index 75d286365..75d286365 100644 --- a/youtube_dl/extractor/myvi.py +++ b/youtube_dlc/extractor/myvi.py diff --git a/youtube_dlc/extractor/myvideoge.py b/youtube_dlc/extractor/myvideoge.py new file mode 100644 index 000000000..0a1d7d0cb --- /dev/null +++ b/youtube_dlc/extractor/myvideoge.py @@ -0,0 +1,56 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import js_to_json + + +class MyVideoGeIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?myvideo\.ge/v/(?P<id>[0-9]+)' + _TEST = { + 'url': 'https://www.myvideo.ge/v/3941048', + 'md5': '8c192a7d2b15454ba4f29dc9c9a52ea9', + 'info_dict': { + 'id': '3941048', + 'ext': 'mp4', + 'title': 'The best prikol', + 'thumbnail': r're:^https?://.*\.jpg$', + 'uploader': 'md5:d72addd357b0dd914e704781f7f777d8', + 'description': 'md5:5c0371f540f5888d603ebfedd46b6df3' + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex(r'<h1[^>]*>([^<]+)</h1>', webpage, 'title') + description = self._og_search_description(webpage) + thumbnail = self._html_search_meta(['og:image'], webpage) + uploader = self._search_regex(r'<a[^>]+class="mv_user_name"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False) + + jwplayer_sources = self._parse_json( + self._search_regex( + r"(?s)jwplayer\(\"mvplayer\"\).setup\(.*?sources: (.*?])", webpage, 'jwplayer sources'), + video_id, transform_source=js_to_json) + + def _formats_key(f): + if f['label'] == 'SD': + return -1 + elif f['label'] == 'HD': + return 1 + else: + return 0 + + jwplayer_sources = sorted(jwplayer_sources, key=_formats_key) + + formats = self._parse_jwplayer_formats(jwplayer_sources, video_id) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'uploader': uploader, + 'formats': formats, + 'thumbnail': thumbnail + } diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dlc/extractor/myvidster.py index 2117d302d..2117d302d 100644 --- a/youtube_dl/extractor/myvidster.py +++ b/youtube_dlc/extractor/myvidster.py diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dlc/extractor/nationalgeographic.py index ee12e2b47..ee12e2b47 100644 --- a/youtube_dl/extractor/nationalgeographic.py +++ b/youtube_dlc/extractor/nationalgeographic.py diff --git a/youtube_dl/extractor/naver.py b/youtube_dlc/extractor/naver.py index 61fc59126..acf53c1ff 100644 --- a/youtube_dl/extractor/naver.py +++ b/youtube_dlc/extractor/naver.py @@ -164,3 +164,88 @@ class NaverIE(NaverBaseIE): 'age_limit': 19 if current_clip.get('adult') else None, }) return info + + +class NaverLiveIE(InfoExtractor): + IE_NAME = 'Naver:live' + _VALID_URL = r'https?://(?:m\.)?tv(?:cast)?\.naver\.com/l/(?P<id>\d+)' + _GEO_BYPASS = False + _TESTS = [{ + 'url': 'https://tv.naver.com/l/52010', + 'info_dict': { + 'id': '52010', + 'ext': 'm3u8', + 'title': '[LIVE] 뉴스특보 : "수도권 거리두기, 2주간 2단계로 조정"', + 'description': 'md5:df7f0c237a5ed5e786ce5c91efbeaab3', + 'channel_id': 'NTV-ytnnews24-0', + 'start_time': 1597026780000, + }, + }, { + 'url': 'https://tv.naver.com/l/51549', + 'info_dict': { + 'id': '51549', + 'ext': 'm3u8', + 'title': '연합뉴스TV - 코로나19 뉴스특보', + 'description': 'md5:c655e82091bc21e413f549c0eaccc481', + 'channel_id': 'NTV-yonhapnewstv-0', + 'start_time': 1596406380000, + }, + }, { + 'url': 'https://tv.naver.com/l/54887', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + page = self._download_webpage(url, video_id, 'Downloading Page', 'Unable to download Page') + secure_url = self._search_regex(r'sApiF:\s+(?:"|\')([^"\']+)', page, 'secureurl') + + info = self._extract_video_info(video_id, secure_url) + info.update({ + 'description': self._og_search_description(page) + }) + + return info + + def _extract_video_info(self, video_id, url): + video_data = self._download_json(url, video_id, headers=self.geo_verification_headers()) + meta = video_data.get('meta') + status = meta.get('status') + + if status == 'CLOSED': + raise ExtractorError('Stream is offline.', expected=True) + elif status != 'OPENED': + raise ExtractorError('Unknown status %s' % status) + + title = meta.get('title') + stream_list = video_data.get('streams') + + if stream_list is None: + raise ExtractorError('Could not get stream data.', expected=True) + + formats = [] + for quality in stream_list: + if not quality.get('url'): + continue + + prop = quality.get('property') + if prop.get('abr'): # This abr doesn't mean Average audio bitrate. + continue + + formats.extend(self._extract_m3u8_formats( + quality.get('url'), video_id, 'm3u8', + m3u8_id=quality.get('qualityId'), live=True + )) + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + 'channel_id': meta.get('channelId'), + 'channel_url': meta.get('channelUrl'), + 'thumbnail': meta.get('imgUrl'), + 'start_time': meta.get('startTime'), + 'categories': [meta.get('categoryId')], + 'is_live': True + } diff --git a/youtube_dl/extractor/nba.py b/youtube_dlc/extractor/nba.py index be295a7a3..be295a7a3 100644 --- a/youtube_dl/extractor/nba.py +++ b/youtube_dlc/extractor/nba.py diff --git a/youtube_dl/extractor/nbc.py b/youtube_dlc/extractor/nbc.py index 6f3cb3003..6f3cb3003 100644 --- a/youtube_dl/extractor/nbc.py +++ b/youtube_dlc/extractor/nbc.py diff --git a/youtube_dl/extractor/ndr.py b/youtube_dlc/extractor/ndr.py index 2447c812e..f3897c71b 100644 --- a/youtube_dl/extractor/ndr.py +++ b/youtube_dlc/extractor/ndr.py @@ -19,14 +19,15 @@ class NDRBaseIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) display_id = next(group for group in mobj.groups() if group) + id = mobj.group('id') webpage = self._download_webpage(url, display_id) - return self._extract_embed(webpage, display_id) + return self._extract_embed(webpage, display_id, id) class NDRIE(NDRBaseIE): IE_NAME = 'ndr' IE_DESC = 'NDR.de - Norddeutscher Rundfunk' - _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html' + _VALID_URL = r'https?://(?:www\.)?(?:daserste\.)?ndr\.de/(?:[^/]+/)*(?P<display_id>[^/?#]+),(?P<id>[\da-z]+)\.html' _TESTS = [{ # httpVideo, same content id 'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html', @@ -86,12 +87,14 @@ class NDRIE(NDRBaseIE): 'only_matching': True, }] - def _extract_embed(self, webpage, display_id): + def _extract_embed(self, webpage, display_id, id): embed_url = self._html_search_meta( 'embedURL', webpage, 'embed URL', default=None) or self._search_regex( r'\bembedUrl["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage, - 'embed URL', group='url') + 'embed URL', fatal=False, group='url') + if embed_url is None: + return self.url_result('ndr:%s' % id, ie=NDREmbedBaseIE.ie_key()) description = self._search_regex( r'<p[^>]+itemprop="description">([^<]+)</p>', webpage, 'description', default=None) or self._og_search_description(webpage) @@ -152,7 +155,7 @@ class NJoyIE(NDRBaseIE): 'only_matching': True, }] - def _extract_embed(self, webpage, display_id): + def _extract_embed(self, webpage, display_id, id): video_id = self._search_regex( r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id') description = self._search_regex( @@ -253,7 +256,7 @@ class NDREmbedBaseIE(InfoExtractor): class NDREmbedIE(NDREmbedBaseIE): IE_NAME = 'ndr:embed' - _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html' + _VALID_URL = r'https?://(?:www\.)?(?:daserste\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html' _TESTS = [{ 'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html', 'md5': '8b9306142fe65bbdefb5ce24edb6b0a9', diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dlc/extractor/ndtv.py index bc3eb9160..bc3eb9160 100644 --- a/youtube_dl/extractor/ndtv.py +++ b/youtube_dlc/extractor/ndtv.py diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dlc/extractor/nerdcubed.py index 9feccc672..9feccc672 100644 --- a/youtube_dl/extractor/nerdcubed.py +++ b/youtube_dlc/extractor/nerdcubed.py diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dlc/extractor/neteasemusic.py index 978a05841..978a05841 100644 --- a/youtube_dl/extractor/neteasemusic.py +++ b/youtube_dlc/extractor/neteasemusic.py diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dlc/extractor/netzkino.py index aec3026b1..aec3026b1 100644 --- a/youtube_dl/extractor/netzkino.py +++ b/youtube_dlc/extractor/netzkino.py diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dlc/extractor/newgrounds.py index 82e7cf522..82e7cf522 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dlc/extractor/newgrounds.py diff --git a/youtube_dl/extractor/newstube.py b/youtube_dlc/extractor/newstube.py index dab4aec44..dab4aec44 100644 --- a/youtube_dl/extractor/newstube.py +++ b/youtube_dlc/extractor/newstube.py diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dlc/extractor/nextmedia.py index 7bd1290bf..7bd1290bf 100644 --- a/youtube_dl/extractor/nextmedia.py +++ b/youtube_dlc/extractor/nextmedia.py diff --git a/youtube_dl/extractor/nexx.py b/youtube_dlc/extractor/nexx.py index 586c1b7eb..586c1b7eb 100644 --- a/youtube_dl/extractor/nexx.py +++ b/youtube_dlc/extractor/nexx.py diff --git a/youtube_dl/extractor/nfl.py b/youtube_dlc/extractor/nfl.py index 460deb162..460deb162 100644 --- a/youtube_dl/extractor/nfl.py +++ b/youtube_dlc/extractor/nfl.py diff --git a/youtube_dl/extractor/nhk.py b/youtube_dlc/extractor/nhk.py index de6a707c4..de6a707c4 100644 --- a/youtube_dl/extractor/nhk.py +++ b/youtube_dlc/extractor/nhk.py diff --git a/youtube_dl/extractor/nhl.py b/youtube_dlc/extractor/nhl.py index eddfe1f37..eddfe1f37 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dlc/extractor/nhl.py diff --git a/youtube_dl/extractor/nick.py b/youtube_dlc/extractor/nick.py index 2e8b302ac..04b98f7bd 100644 --- a/youtube_dl/extractor/nick.py +++ b/youtube_dlc/extractor/nick.py @@ -245,5 +245,5 @@ class NickRuIE(MTVServicesInfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - mgid = self._extract_mgid(webpage) + mgid = self._extract_mgid(webpage, url) return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid) diff --git a/youtube_dl/extractor/niconico.py b/youtube_dlc/extractor/niconico.py index eb07ca776..eb07ca776 100644 --- a/youtube_dl/extractor/niconico.py +++ b/youtube_dlc/extractor/niconico.py diff --git a/youtube_dl/extractor/ninecninemedia.py b/youtube_dlc/extractor/ninecninemedia.py index 65754c5e7..65754c5e7 100644 --- a/youtube_dl/extractor/ninecninemedia.py +++ b/youtube_dlc/extractor/ninecninemedia.py diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dlc/extractor/ninegag.py index dc6a27d36..dc6a27d36 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dlc/extractor/ninegag.py diff --git a/youtube_dl/extractor/ninenow.py b/youtube_dlc/extractor/ninenow.py index 6157dc7c1..6157dc7c1 100644 --- a/youtube_dl/extractor/ninenow.py +++ b/youtube_dlc/extractor/ninenow.py diff --git a/youtube_dl/extractor/nintendo.py b/youtube_dlc/extractor/nintendo.py index ff8f70ba6..ff8f70ba6 100644 --- a/youtube_dl/extractor/nintendo.py +++ b/youtube_dlc/extractor/nintendo.py diff --git a/youtube_dl/extractor/njpwworld.py b/youtube_dlc/extractor/njpwworld.py index 025c5d249..025c5d249 100644 --- a/youtube_dl/extractor/njpwworld.py +++ b/youtube_dlc/extractor/njpwworld.py diff --git a/youtube_dl/extractor/nobelprize.py b/youtube_dlc/extractor/nobelprize.py index 4dfdb09d6..4dfdb09d6 100644 --- a/youtube_dl/extractor/nobelprize.py +++ b/youtube_dlc/extractor/nobelprize.py diff --git a/youtube_dl/extractor/noco.py b/youtube_dlc/extractor/noco.py index 30df905af..30df905af 100644 --- a/youtube_dl/extractor/noco.py +++ b/youtube_dlc/extractor/noco.py diff --git a/youtube_dl/extractor/nonktube.py b/youtube_dlc/extractor/nonktube.py index ca1424e06..ca1424e06 100644 --- a/youtube_dl/extractor/nonktube.py +++ b/youtube_dlc/extractor/nonktube.py diff --git a/youtube_dl/extractor/noovo.py b/youtube_dlc/extractor/noovo.py index b40770d07..b40770d07 100644 --- a/youtube_dl/extractor/noovo.py +++ b/youtube_dlc/extractor/noovo.py diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dlc/extractor/normalboots.py index 61fe571df..61fe571df 100644 --- a/youtube_dl/extractor/normalboots.py +++ b/youtube_dlc/extractor/normalboots.py diff --git a/youtube_dl/extractor/nosvideo.py b/youtube_dlc/extractor/nosvideo.py index 53c500c35..53c500c35 100644 --- a/youtube_dl/extractor/nosvideo.py +++ b/youtube_dlc/extractor/nosvideo.py diff --git a/youtube_dl/extractor/nova.py b/youtube_dlc/extractor/nova.py index 47b9748f0..47b9748f0 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dlc/extractor/nova.py diff --git a/youtube_dl/extractor/nowness.py b/youtube_dlc/extractor/nowness.py index f26dafb8f..c136bc8c0 100644 --- a/youtube_dl/extractor/nowness.py +++ b/youtube_dlc/extractor/nowness.py @@ -37,7 +37,7 @@ class NownessBaseIE(InfoExtractor): elif source == 'youtube': return self.url_result(video_id, 'Youtube') elif source == 'cinematique': - # youtube-dl currently doesn't support cinematique + # youtube-dlc currently doesn't support cinematique # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique') pass diff --git a/youtube_dl/extractor/noz.py b/youtube_dlc/extractor/noz.py index ccafd7723..ccafd7723 100644 --- a/youtube_dl/extractor/noz.py +++ b/youtube_dlc/extractor/noz.py diff --git a/youtube_dl/extractor/npo.py b/youtube_dlc/extractor/npo.py index e525ad928..e525ad928 100644 --- a/youtube_dl/extractor/npo.py +++ b/youtube_dlc/extractor/npo.py diff --git a/youtube_dl/extractor/npr.py b/youtube_dlc/extractor/npr.py index 53acc6e57..53acc6e57 100644 --- a/youtube_dl/extractor/npr.py +++ b/youtube_dlc/extractor/npr.py diff --git a/youtube_dl/extractor/nrk.py b/youtube_dlc/extractor/nrk.py index 84aacbcda..84aacbcda 100644 --- a/youtube_dl/extractor/nrk.py +++ b/youtube_dlc/extractor/nrk.py diff --git a/youtube_dl/extractor/nrl.py b/youtube_dlc/extractor/nrl.py index 22a2df8d3..22a2df8d3 100644 --- a/youtube_dl/extractor/nrl.py +++ b/youtube_dlc/extractor/nrl.py diff --git a/youtube_dl/extractor/ntvcojp.py b/youtube_dlc/extractor/ntvcojp.py index 0c8221b22..0c8221b22 100644 --- a/youtube_dl/extractor/ntvcojp.py +++ b/youtube_dlc/extractor/ntvcojp.py diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dlc/extractor/ntvde.py index 101a5374c..101a5374c 100644 --- a/youtube_dl/extractor/ntvde.py +++ b/youtube_dlc/extractor/ntvde.py diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dlc/extractor/ntvru.py index c47d1dfa4..c47d1dfa4 100644 --- a/youtube_dl/extractor/ntvru.py +++ b/youtube_dlc/extractor/ntvru.py diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dlc/extractor/nuevo.py index be1e09d37..be1e09d37 100644 --- a/youtube_dl/extractor/nuevo.py +++ b/youtube_dlc/extractor/nuevo.py diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dlc/extractor/nuvid.py index ab6bfcd7f..ab6bfcd7f 100644 --- a/youtube_dl/extractor/nuvid.py +++ b/youtube_dlc/extractor/nuvid.py diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dlc/extractor/nytimes.py index fc78ca56c..fc78ca56c 100644 --- a/youtube_dl/extractor/nytimes.py +++ b/youtube_dlc/extractor/nytimes.py diff --git a/youtube_dl/extractor/nzz.py b/youtube_dlc/extractor/nzz.py index 61ee77adb..61ee77adb 100644 --- a/youtube_dl/extractor/nzz.py +++ b/youtube_dlc/extractor/nzz.py diff --git a/youtube_dl/extractor/odatv.py b/youtube_dlc/extractor/odatv.py index 314527f98..314527f98 100644 --- a/youtube_dl/extractor/odatv.py +++ b/youtube_dlc/extractor/odatv.py diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dlc/extractor/odnoklassniki.py index 7ed9fac55..7ed9fac55 100644 --- a/youtube_dl/extractor/odnoklassniki.py +++ b/youtube_dlc/extractor/odnoklassniki.py diff --git a/youtube_dl/extractor/oktoberfesttv.py b/youtube_dlc/extractor/oktoberfesttv.py index a914068f9..a914068f9 100644 --- a/youtube_dl/extractor/oktoberfesttv.py +++ b/youtube_dlc/extractor/oktoberfesttv.py diff --git a/youtube_dl/extractor/once.py b/youtube_dlc/extractor/once.py index 3e44b7829..3e44b7829 100644 --- a/youtube_dl/extractor/once.py +++ b/youtube_dlc/extractor/once.py diff --git a/youtube_dl/extractor/ondemandkorea.py b/youtube_dlc/extractor/ondemandkorea.py index df1ce3c1d..cc3c587bc 100644 --- a/youtube_dl/extractor/ondemandkorea.py +++ b/youtube_dlc/extractor/ondemandkorea.py @@ -11,18 +11,34 @@ from ..utils import ( class OnDemandKoreaIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?ondemandkorea\.com/(?P<id>[^/]+)\.html' _GEO_COUNTRIES = ['US', 'CA'] - _TEST = { - 'url': 'http://www.ondemandkorea.com/ask-us-anything-e43.html', + _TESTS = [{ + 'url': 'https://www.ondemandkorea.com/ask-us-anything-e43.html', 'info_dict': { 'id': 'ask-us-anything-e43', 'ext': 'mp4', - 'title': 'Ask Us Anything : E43', + 'title': 'Ask Us Anything : Gain, Ji Soo - 09/24/2016', + 'description': 'A talk show/game show with a school theme where celebrity guests appear as “transfer students.”', 'thumbnail': r're:^https?://.*\.jpg$', }, 'params': { 'skip_download': 'm3u8 download' } - } + }, { + 'url': 'https://www.ondemandkorea.com/confession-e01-1.html', + 'info_dict': { + 'id': 'confession-e01-1', + 'ext': 'mp4', + 'title': 'Confession : E01', + 'description': 'Choi Do-hyun, a criminal attorney, is the son of a death row convict. Ever since Choi Pil-su got arrested for murder, Do-hyun has wanted to solve his ', + 'thumbnail': r're:^https?://.*\.jpg$', + 'subtitles': { + 'English': 'mincount:1', + }, + }, + 'params': { + 'skip_download': 'm3u8 download' + } + }] def _real_extract(self, url): video_id = self._match_id(url) @@ -44,11 +60,18 @@ class OnDemandKoreaIE(InfoExtractor): 'This video is only available to ODK PLUS members.', expected=True) - title = self._og_search_title(webpage) + if 'ODK PREMIUM Members Only' in webpage: + raise ExtractorError( + 'This video is only available to ODK PREMIUM members.', + expected=True) + + title = self._search_regex( + r'class=["\']episode_title["\'][^>]*>([^<]+)', + webpage, 'episode_title', fatal=False) or self._og_search_title(webpage) jw_config = self._parse_json( self._search_regex( - r'(?s)jwplayer\(([\'"])(?:(?!\1).)+\1\)\.setup\s*\((?P<options>.+?)\);', + r'(?s)odkPlayer\.init.*?(?P<options>{[^;]+}).*?;', webpage, 'jw config', group='options'), video_id, transform_source=js_to_json) info = self._parse_jwplayer_data( @@ -57,6 +80,7 @@ class OnDemandKoreaIE(InfoExtractor): info.update({ 'title': title, - 'thumbnail': self._og_search_thumbnail(webpage), + 'description': self._og_search_description(webpage), + 'thumbnail': self._og_search_thumbnail(webpage) }) return info diff --git a/youtube_dl/extractor/onet.py b/youtube_dlc/extractor/onet.py index e55b2ac89..e55b2ac89 100644 --- a/youtube_dl/extractor/onet.py +++ b/youtube_dlc/extractor/onet.py diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dlc/extractor/onionstudios.py index cf5c39e66..cf5c39e66 100644 --- a/youtube_dl/extractor/onionstudios.py +++ b/youtube_dlc/extractor/onionstudios.py diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dlc/extractor/ooyala.py index eb957b8fe..eb957b8fe 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dlc/extractor/ooyala.py diff --git a/youtube_dl/extractor/openload.py b/youtube_dlc/extractor/openload.py index 0c20d0177..0c20d0177 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dlc/extractor/openload.py diff --git a/youtube_dl/extractor/ora.py b/youtube_dlc/extractor/ora.py index 1d42be39b..1d42be39b 100644 --- a/youtube_dl/extractor/ora.py +++ b/youtube_dlc/extractor/ora.py diff --git a/youtube_dl/extractor/orf.py b/youtube_dlc/extractor/orf.py index 700ce448c..700ce448c 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dlc/extractor/orf.py diff --git a/youtube_dl/extractor/outsidetv.py b/youtube_dlc/extractor/outsidetv.py index c5333b08c..c5333b08c 100644 --- a/youtube_dl/extractor/outsidetv.py +++ b/youtube_dlc/extractor/outsidetv.py diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dlc/extractor/packtpub.py index 11ad3b3b8..11ad3b3b8 100644 --- a/youtube_dl/extractor/packtpub.py +++ b/youtube_dlc/extractor/packtpub.py diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dlc/extractor/pandoratv.py index 538738c09..538738c09 100644 --- a/youtube_dl/extractor/pandoratv.py +++ b/youtube_dlc/extractor/pandoratv.py diff --git a/youtube_dl/extractor/parliamentliveuk.py b/youtube_dlc/extractor/parliamentliveuk.py index bdd5ff565..bdd5ff565 100644 --- a/youtube_dl/extractor/parliamentliveuk.py +++ b/youtube_dlc/extractor/parliamentliveuk.py diff --git a/youtube_dl/extractor/patreon.py b/youtube_dlc/extractor/patreon.py index 761a4b1de..761a4b1de 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dlc/extractor/patreon.py diff --git a/youtube_dl/extractor/pbs.py b/youtube_dlc/extractor/pbs.py index 4dbe661be..4dbe661be 100644 --- a/youtube_dl/extractor/pbs.py +++ b/youtube_dlc/extractor/pbs.py diff --git a/youtube_dl/extractor/pearvideo.py b/youtube_dlc/extractor/pearvideo.py index 1d777221c..1d777221c 100644 --- a/youtube_dl/extractor/pearvideo.py +++ b/youtube_dlc/extractor/pearvideo.py diff --git a/youtube_dl/extractor/peertube.py b/youtube_dlc/extractor/peertube.py index 48fb95416..48fb95416 100644 --- a/youtube_dl/extractor/peertube.py +++ b/youtube_dlc/extractor/peertube.py diff --git a/youtube_dl/extractor/people.py b/youtube_dlc/extractor/people.py index 6ca95715e..6ca95715e 100644 --- a/youtube_dl/extractor/people.py +++ b/youtube_dlc/extractor/people.py diff --git a/youtube_dl/extractor/performgroup.py b/youtube_dlc/extractor/performgroup.py index 26942bfb3..26942bfb3 100644 --- a/youtube_dl/extractor/performgroup.py +++ b/youtube_dlc/extractor/performgroup.py diff --git a/youtube_dl/extractor/periscope.py b/youtube_dlc/extractor/periscope.py index b15906390..b15906390 100644 --- a/youtube_dl/extractor/periscope.py +++ b/youtube_dlc/extractor/periscope.py diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dlc/extractor/philharmoniedeparis.py index 03da64b11..03da64b11 100644 --- a/youtube_dl/extractor/philharmoniedeparis.py +++ b/youtube_dlc/extractor/philharmoniedeparis.py diff --git a/youtube_dlc/extractor/phoenix.py b/youtube_dlc/extractor/phoenix.py new file mode 100644 index 000000000..8d52ad3b4 --- /dev/null +++ b/youtube_dlc/extractor/phoenix.py @@ -0,0 +1,52 @@ +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ExtractorError + + +class PhoenixIE(InfoExtractor): + IE_NAME = 'phoenix.de' + _VALID_URL = r'''https?://(?:www\.)?phoenix.de/\D+(?P<id>\d+)\.html''' + _TESTS = [ + { + 'url': 'https://www.phoenix.de/sendungen/dokumentationen/unsere-welt-in-zukunft---stadt-a-1283620.html', + 'md5': '5e765e838aa3531c745a4f5b249ee3e3', + 'info_dict': { + 'id': '0OB4HFc43Ns', + 'ext': 'mp4', + 'title': 'Unsere Welt in Zukunft - Stadt', + 'description': 'md5:9bfb6fd498814538f953b2dcad7ce044', + 'upload_date': '20190912', + 'uploader': 'phoenix', + 'uploader_id': 'phoenix', + } + }, + { + 'url': 'https://www.phoenix.de/drohnenangriffe-in-saudi-arabien-a-1286995.html?ref=aktuelles', + 'only_matching': True, + }, + # an older page: https://www.phoenix.de/sendungen/gespraeche/phoenix-persoenlich/im-dialog-a-177727.html + # seems to not have an embedded video, even though it's uploaded on youtube: https://www.youtube.com/watch?v=4GxnoUHvOkM + ] + + def extract_from_json_api(self, video_id, api_url): + doc = self._download_json( + api_url, video_id, + note="Downloading webpage metadata", + errnote="Failed to load webpage metadata") + + for a in doc["absaetze"]: + if a["typ"] == "video-youtube": + return { + '_type': 'url_transparent', + 'id': a["id"], + 'title': doc["titel"], + 'url': "https://www.youtube.com/watch?v=%s" % a["id"], + 'ie_key': 'Youtube', + } + raise ExtractorError("No downloadable video found", expected=True) + + def _real_extract(self, url): + page_id = self._match_id(url) + api_url = 'https://www.phoenix.de/response/id/%s' % page_id + return self.extract_from_json_api(page_id, api_url) diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dlc/extractor/photobucket.py index 6c8bbe1d9..6c8bbe1d9 100644 --- a/youtube_dl/extractor/photobucket.py +++ b/youtube_dlc/extractor/photobucket.py diff --git a/youtube_dl/extractor/picarto.py b/youtube_dlc/extractor/picarto.py index 8099ef1d6..8099ef1d6 100644 --- a/youtube_dl/extractor/picarto.py +++ b/youtube_dlc/extractor/picarto.py diff --git a/youtube_dl/extractor/piksel.py b/youtube_dlc/extractor/piksel.py index 88b6859b0..88b6859b0 100644 --- a/youtube_dl/extractor/piksel.py +++ b/youtube_dlc/extractor/piksel.py diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dlc/extractor/pinkbike.py index 9f3501f77..9f3501f77 100644 --- a/youtube_dl/extractor/pinkbike.py +++ b/youtube_dlc/extractor/pinkbike.py diff --git a/youtube_dl/extractor/pladform.py b/youtube_dlc/extractor/pladform.py index e86c65396..e86c65396 100644 --- a/youtube_dl/extractor/pladform.py +++ b/youtube_dlc/extractor/pladform.py diff --git a/youtube_dl/extractor/platzi.py b/youtube_dlc/extractor/platzi.py index 23c8256b5..23c8256b5 100644 --- a/youtube_dl/extractor/platzi.py +++ b/youtube_dlc/extractor/platzi.py diff --git a/youtube_dl/extractor/playfm.py b/youtube_dlc/extractor/playfm.py index e766ccca3..e766ccca3 100644 --- a/youtube_dl/extractor/playfm.py +++ b/youtube_dlc/extractor/playfm.py diff --git a/youtube_dl/extractor/playplustv.py b/youtube_dlc/extractor/playplustv.py index 1e30ab23a..1e30ab23a 100644 --- a/youtube_dl/extractor/playplustv.py +++ b/youtube_dlc/extractor/playplustv.py diff --git a/youtube_dl/extractor/plays.py b/youtube_dlc/extractor/plays.py index ddfc6f148..ddfc6f148 100644 --- a/youtube_dl/extractor/plays.py +++ b/youtube_dlc/extractor/plays.py diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dlc/extractor/playtvak.py index 4c5f57919..4c5f57919 100644 --- a/youtube_dl/extractor/playtvak.py +++ b/youtube_dlc/extractor/playtvak.py diff --git a/youtube_dl/extractor/playvid.py b/youtube_dlc/extractor/playvid.py index 4aef186ea..4aef186ea 100644 --- a/youtube_dl/extractor/playvid.py +++ b/youtube_dlc/extractor/playvid.py diff --git a/youtube_dl/extractor/playwire.py b/youtube_dlc/extractor/playwire.py index 4d96a10a7..4d96a10a7 100644 --- a/youtube_dl/extractor/playwire.py +++ b/youtube_dlc/extractor/playwire.py diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dlc/extractor/pluralsight.py index abd08bc28..abd08bc28 100644 --- a/youtube_dl/extractor/pluralsight.py +++ b/youtube_dlc/extractor/pluralsight.py diff --git a/youtube_dl/extractor/podomatic.py b/youtube_dlc/extractor/podomatic.py index e782e3f1f..e782e3f1f 100644 --- a/youtube_dl/extractor/podomatic.py +++ b/youtube_dlc/extractor/podomatic.py diff --git a/youtube_dl/extractor/pokemon.py b/youtube_dlc/extractor/pokemon.py index 80222d428..14ee1a72e 100644 --- a/youtube_dl/extractor/pokemon.py +++ b/youtube_dlc/extractor/pokemon.py @@ -5,8 +5,11 @@ import re from .common import InfoExtractor from ..utils import ( + ExtractorError, extract_attributes, int_or_none, + js_to_json, + merge_dicts, ) @@ -69,3 +72,67 @@ class PokemonIE(InfoExtractor): 'episode_number': int_or_none(video_data.get('data-video-episode')), 'ie_key': 'LimelightMedia', } + + +class PokemonWatchIE(InfoExtractor): + _VALID_URL = r'https?://watch\.pokemon\.com/[a-z]{2}-[a-z]{2}/player\.html\?id=(?P<id>[a-z0-9]{32})' + _API_URL = 'https://www.pokemon.com/api/pokemontv/v2/channels/{0:}' + _TESTS = [{ + 'url': 'https://watch.pokemon.com/en-us/player.html?id=8309a40969894a8e8d5bc1311e9c5667', + 'md5': '62833938a31e61ab49ada92f524c42ff', + 'info_dict': { + 'id': '8309a40969894a8e8d5bc1311e9c5667', + 'ext': 'mp4', + 'title': 'Lillier and the Staff!', + 'description': 'md5:338841b8c21b283d24bdc9b568849f04', + } + }, { + 'url': 'https://watch.pokemon.com/de-de/player.html?id=b3c402e111a4459eb47e12160ab0ba07', + 'only_matching': True + }] + + def _extract_media(self, channel_array, video_id): + for channel in channel_array: + for media in channel.get('media'): + if media.get('id') == video_id: + return media + return None + + def _real_extract(self, url): + video_id = self._match_id(url) + + info = { + '_type': 'url', + 'id': video_id, + 'url': 'limelight:media:%s' % video_id, + 'ie_key': 'LimelightMedia', + } + + # API call can be avoided entirely if we are listing formats + if self._downloader.params.get('listformats', False): + return info + + webpage = self._download_webpage(url, video_id) + build_vars = self._parse_json(self._search_regex( + r'(?s)buildVars\s*=\s*({.*?})', webpage, 'build vars'), + video_id, transform_source=js_to_json) + region = build_vars.get('region') + channel_array = self._download_json(self._API_URL.format(region), video_id) + video_data = self._extract_media(channel_array, video_id) + + if video_data is None: + raise ExtractorError( + 'Video %s does not exist' % video_id, expected=True) + + info['_type'] = 'url_transparent' + images = video_data.get('images') + + return merge_dicts(info, { + 'title': video_data.get('title'), + 'description': video_data.get('description'), + 'thumbnail': images.get('medium') or images.get('small'), + 'series': 'Pokémon', + 'season_number': int_or_none(video_data.get('season')), + 'episode': video_data.get('title'), + 'episode_number': int_or_none(video_data.get('episode')), + }) diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dlc/extractor/polskieradio.py index 978d6f813..978d6f813 100644 --- a/youtube_dl/extractor/polskieradio.py +++ b/youtube_dlc/extractor/polskieradio.py diff --git a/youtube_dl/extractor/popcorntimes.py b/youtube_dlc/extractor/popcorntimes.py index 7bf7f9858..7bf7f9858 100644 --- a/youtube_dl/extractor/popcorntimes.py +++ b/youtube_dlc/extractor/popcorntimes.py diff --git a/youtube_dl/extractor/popcorntv.py b/youtube_dlc/extractor/popcorntv.py index 9f834fb6c..9f834fb6c 100644 --- a/youtube_dl/extractor/popcorntv.py +++ b/youtube_dlc/extractor/popcorntv.py diff --git a/youtube_dl/extractor/porn91.py b/youtube_dlc/extractor/porn91.py index 20eac647a..20eac647a 100644 --- a/youtube_dl/extractor/porn91.py +++ b/youtube_dlc/extractor/porn91.py diff --git a/youtube_dl/extractor/porncom.py b/youtube_dlc/extractor/porncom.py index 5726cab3a..5726cab3a 100644 --- a/youtube_dl/extractor/porncom.py +++ b/youtube_dlc/extractor/porncom.py diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dlc/extractor/pornhd.py index c6052ac9f..c6052ac9f 100644 --- a/youtube_dl/extractor/pornhd.py +++ b/youtube_dlc/extractor/pornhd.py diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dlc/extractor/pornhub.py index 3567a3283..529f3f711 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dlc/extractor/pornhub.py @@ -17,6 +17,7 @@ from ..utils import ( determine_ext, ExtractorError, int_or_none, + merge_dicts, NO_DEFAULT, orderedSet, remove_quotes, @@ -59,13 +60,14 @@ class PornHubIE(PornHubBaseIE): ''' _TESTS = [{ 'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015', - 'md5': '1e19b41231a02eba417839222ac9d58e', + 'md5': 'a6391306d050e4547f62b3f485dd9ba9', 'info_dict': { 'id': '648719015', 'ext': 'mp4', 'title': 'Seductive Indian beauty strips down and fingers her pink pussy', 'uploader': 'Babes', 'upload_date': '20130628', + 'timestamp': 1372447216, 'duration': 361, 'view_count': int, 'like_count': int, @@ -82,8 +84,8 @@ class PornHubIE(PornHubBaseIE): 'id': '1331683002', 'ext': 'mp4', 'title': '重庆婷婷女王足交', - 'uploader': 'Unknown', 'upload_date': '20150213', + 'timestamp': 1423804862, 'duration': 1753, 'view_count': int, 'like_count': int, @@ -121,6 +123,7 @@ class PornHubIE(PornHubBaseIE): 'params': { 'skip_download': True, }, + 'skip': 'This video has been disabled', }, { 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d', 'only_matching': True, @@ -338,10 +341,10 @@ class PornHubIE(PornHubBaseIE): video_uploader = self._html_search_regex( r'(?s)From: .+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<', - webpage, 'uploader', fatal=False) + webpage, 'uploader', default=None) view_count = self._extract_count( - r'<span class="count">([\d,\.]+)</span> views', webpage, 'view') + r'<span class="count">([\d,\.]+)</span> [Vv]iews', webpage, 'view') like_count = self._extract_count( r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like') dislike_count = self._extract_count( @@ -356,7 +359,11 @@ class PornHubIE(PornHubBaseIE): if div: return re.findall(r'<a[^>]+\bhref=[^>]+>([^<]+)', div) - return { + info = self._search_json_ld(webpage, video_id, default={}) + # description provided in JSON-LD is irrelevant + info['description'] = None + + return merge_dicts({ 'id': video_id, 'uploader': video_uploader, 'upload_date': upload_date, @@ -372,7 +379,7 @@ class PornHubIE(PornHubBaseIE): 'tags': extract_list('tags'), 'categories': extract_list('categories'), 'subtitles': subtitles, - } + }, info) class PornHubPlaylistBaseIE(PornHubBaseIE): diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dlc/extractor/pornotube.py index 1b5b9a320..1b5b9a320 100644 --- a/youtube_dl/extractor/pornotube.py +++ b/youtube_dlc/extractor/pornotube.py diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dlc/extractor/pornovoisines.py index b6b71069d..b6b71069d 100644 --- a/youtube_dl/extractor/pornovoisines.py +++ b/youtube_dlc/extractor/pornovoisines.py diff --git a/youtube_dl/extractor/pornoxo.py b/youtube_dlc/extractor/pornoxo.py index 2831368b6..2831368b6 100644 --- a/youtube_dl/extractor/pornoxo.py +++ b/youtube_dlc/extractor/pornoxo.py diff --git a/youtube_dl/extractor/presstv.py b/youtube_dlc/extractor/presstv.py index b5c279203..b5c279203 100644 --- a/youtube_dl/extractor/presstv.py +++ b/youtube_dlc/extractor/presstv.py diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dlc/extractor/prosiebensat1.py index e47088292..e47088292 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dlc/extractor/prosiebensat1.py diff --git a/youtube_dl/extractor/puhutv.py b/youtube_dlc/extractor/puhutv.py index ca71665e0..ca71665e0 100644 --- a/youtube_dl/extractor/puhutv.py +++ b/youtube_dlc/extractor/puhutv.py diff --git a/youtube_dl/extractor/puls4.py b/youtube_dlc/extractor/puls4.py index 80091b85f..80091b85f 100644 --- a/youtube_dl/extractor/puls4.py +++ b/youtube_dlc/extractor/puls4.py diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dlc/extractor/pyvideo.py index b8ac93a62..b8ac93a62 100644 --- a/youtube_dl/extractor/pyvideo.py +++ b/youtube_dlc/extractor/pyvideo.py diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dlc/extractor/qqmusic.py index 084308aeb..084308aeb 100644 --- a/youtube_dl/extractor/qqmusic.py +++ b/youtube_dlc/extractor/qqmusic.py diff --git a/youtube_dl/extractor/r7.py b/youtube_dlc/extractor/r7.py index e2202d603..e2202d603 100644 --- a/youtube_dl/extractor/r7.py +++ b/youtube_dlc/extractor/r7.py diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dlc/extractor/radiobremen.py index 2c35f9845..2c35f9845 100644 --- a/youtube_dl/extractor/radiobremen.py +++ b/youtube_dlc/extractor/radiobremen.py diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dlc/extractor/radiocanada.py index a28b1a24c..a28b1a24c 100644 --- a/youtube_dl/extractor/radiocanada.py +++ b/youtube_dlc/extractor/radiocanada.py diff --git a/youtube_dl/extractor/radiode.py b/youtube_dlc/extractor/radiode.py index 2c06c8b1e..2c06c8b1e 100644 --- a/youtube_dl/extractor/radiode.py +++ b/youtube_dlc/extractor/radiode.py diff --git a/youtube_dl/extractor/radiofrance.py b/youtube_dlc/extractor/radiofrance.py index a8afc0014..a8afc0014 100644 --- a/youtube_dl/extractor/radiofrance.py +++ b/youtube_dlc/extractor/radiofrance.py diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dlc/extractor/radiojavan.py index 3f74f0c01..3f74f0c01 100644 --- a/youtube_dl/extractor/radiojavan.py +++ b/youtube_dlc/extractor/radiojavan.py diff --git a/youtube_dl/extractor/rai.py b/youtube_dlc/extractor/rai.py index 207a6c247..51a310f5c 100644 --- a/youtube_dl/extractor/rai.py +++ b/youtube_dlc/extractor/rai.py @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import unicode_literals import re @@ -17,7 +18,6 @@ from ..utils import ( parse_duration, strip_or_none, try_get, - unescapeHTML, unified_strdate, unified_timestamp, update_url_query, @@ -30,6 +30,7 @@ class RaiBaseIE(InfoExtractor): _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}' _GEO_COUNTRIES = ['IT'] _GEO_BYPASS = False + _BASE_URL = 'https://www.raiplay.it' def _extract_relinker_info(self, relinker_url, video_id): if not re.match(r'https?://', relinker_url): @@ -122,41 +123,19 @@ class RaiBaseIE(InfoExtractor): class RaiPlayIE(RaiBaseIE): - _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE + _VALID_URL = r'(?P<url>(?P<base>https?://(?:www\.)?raiplay\.it/.+?-)(?P<id>%s)(?P<ext>\.(?:html|json)))' % RaiBaseIE._UUID_RE _TESTS = [{ - 'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter', - 'md5': '340aa3b7afb54bfd14a8c11786450d76', - 'info_dict': { - 'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66', - 'ext': 'mp4', - 'title': 'La Casa Bianca', - 'alt_title': 'S2016 - Puntata del 23/10/2016', - 'description': 'md5:a09d45890850458077d1f68bb036e0a5', - 'thumbnail': r're:^https?://.*\.jpg$', - 'uploader': 'Rai 3', - 'creator': 'Rai 3', - 'duration': 3278, - 'timestamp': 1477764300, - 'upload_date': '20161029', - 'series': 'La Casa Bianca', - 'season': '2016', - }, - }, { 'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html', 'md5': '8970abf8caf8aef4696e7b1f2adfc696', 'info_dict': { 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391', 'ext': 'mp4', 'title': 'Report del 07/04/2014', - 'alt_title': 'S2013/14 - Puntata del 07/04/2014', - 'description': 'md5:f27c544694cacb46a078db84ec35d2d9', + 'alt_title': 'St 2013/14 - Espresso nel caffè - 07/04/2014 ', + 'description': 'md5:d730c168a58f4bb35600fc2f881ec04e', 'thumbnail': r're:^https?://.*\.jpg$', - 'uploader': 'Rai 5', - 'creator': 'Rai 5', + 'uploader': 'Rai Gulp', 'duration': 6160, - 'series': 'Report', - 'season_number': 5, - 'season': '2013/14', }, 'params': { 'skip_download': True, @@ -168,16 +147,15 @@ class RaiPlayIE(RaiBaseIE): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - url, video_id = mobj.group('url', 'id') + url, base, video_id, ext = mobj.group('url', 'base', 'id', 'ext') media = self._download_json( - '%s?json' % url, video_id, 'Downloading video JSON') + '%s%s.json' % (base, video_id), video_id, 'Downloading video JSON') title = media['name'] - video = media['video'] - relinker_info = self._extract_relinker_info(video['contentUrl'], video_id) + relinker_info = self._extract_relinker_info(video['content_url'], video_id) self._sort_formats(relinker_info['formats']) thumbnails = [] @@ -185,7 +163,7 @@ class RaiPlayIE(RaiBaseIE): for _, value in media.get('images').items(): if value: thumbnails.append({ - 'url': value.replace('[RESOLUTION]', '600x400') + 'url': urljoin(RaiBaseIE._BASE_URL, value.replace('[RESOLUTION]', '600x400')) }) timestamp = unified_timestamp(try_get( @@ -225,7 +203,7 @@ class RaiPlayLiveIE(RaiBaseIE): 'display_id': 'rainews24', 'ext': 'mp4', 'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', - 'description': 'md5:6eca31500550f9376819f174e5644754', + 'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497', 'uploader': 'Rai News 24', 'creator': 'Rai News 24', 'is_live': True, @@ -238,20 +216,32 @@ class RaiPlayLiveIE(RaiBaseIE): def _real_extract(self, url): display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) + media = self._download_json( + '%s.json' % urljoin(RaiBaseIE._BASE_URL, 'dirette/' + display_id), + display_id, 'Downloading channel JSON') + + title = media['name'] + video = media['video'] + video_id = media['id'].replace('ContentItem-', '') - video_id = self._search_regex( - r'data-uniquename=["\']ContentItem-(%s)' % RaiBaseIE._UUID_RE, - webpage, 'content id') + relinker_info = self._extract_relinker_info(video['content_url'], video_id) + self._sort_formats(relinker_info['formats']) - return { - '_type': 'url_transparent', - 'ie_key': RaiPlayIE.ie_key(), - 'url': 'http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id, + info = { 'id': video_id, 'display_id': display_id, + 'title': self._live_title(title) if relinker_info.get( + 'is_live') else title, + 'alt_title': media.get('subtitle'), + 'description': media.get('description'), + 'uploader': strip_or_none(media.get('channel')), + 'creator': strip_or_none(media.get('editor')), + 'duration': parse_duration(video.get('duration')), } + info.update(relinker_info) + return info + class RaiPlayPlaylistIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+)' @@ -260,7 +250,7 @@ class RaiPlayPlaylistIE(InfoExtractor): 'info_dict': { 'id': 'nondirloalmiocapo', 'title': 'Non dirlo al mio capo', - 'description': 'md5:9f3d603b2947c1c7abb098f3b14fac86', + 'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b', }, 'playlist_mincount': 12, }] @@ -268,21 +258,25 @@ class RaiPlayPlaylistIE(InfoExtractor): def _real_extract(self, url): playlist_id = self._match_id(url) - webpage = self._download_webpage(url, playlist_id) + media = self._download_json( + '%s.json' % urljoin(RaiBaseIE._BASE_URL, 'programmi/' + playlist_id), + playlist_id, 'Downloading program JSON') - title = self._html_search_meta( - ('programma', 'nomeProgramma'), webpage, 'title') - description = unescapeHTML(self._html_search_meta( - ('description', 'og:description'), webpage, 'description')) + title = media['name'] + description = media['program_info']['description'] + + content_sets = [s['id'] for b in media['blocks'] for s in b['sets']] entries = [] - for mobj in re.finditer( - r'<a\b[^>]+\bhref=(["\'])(?P<path>/raiplay/video/.+?)\1', - webpage): - video_url = urljoin(url, mobj.group('path')) - entries.append(self.url_result( - video_url, ie=RaiPlayIE.ie_key(), - video_id=RaiPlayIE._match_id(video_url))) + for cs in content_sets: + medias = self._download_json( + '%s/%s.json' % (urljoin(RaiBaseIE._BASE_URL, 'programmi/' + playlist_id), cs), + cs, 'Downloading content set JSON') + for m in medias['items']: + video_url = urljoin(url, m['path_id']) + entries.append(self.url_result( + video_url, ie=RaiPlayIE.ie_key(), + video_id=RaiPlayIE._match_id(video_url))) return self.playlist_result(entries, playlist_id, title, description) @@ -316,7 +310,7 @@ class RaiIE(RaiBaseIE): }, { # with ContentItem in og:url 'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html', - 'md5': '11959b4e44fa74de47011b5799490adf', + 'md5': '6865dd00cf0bbf5772fdd89d59bd768a', 'info_dict': { 'id': 'efb17665-691c-45d5-a60c-5301333cbb0c', 'ext': 'mp4', @@ -327,18 +321,6 @@ class RaiIE(RaiBaseIE): 'upload_date': '20161103', } }, { - # drawMediaRaiTV(...) - 'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html', - 'md5': '2dd727e61114e1ee9c47f0da6914e178', - 'info_dict': { - 'id': '59d69d28-6bb6-409d-a4b5-ed44096560af', - 'ext': 'mp4', - 'title': 'Il pacco', - 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a', - 'thumbnail': r're:^https?://.*\.jpg$', - 'upload_date': '20141221', - }, - }, { # initEdizione('ContentItem-...' 'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined', 'info_dict': { @@ -350,17 +332,6 @@ class RaiIE(RaiBaseIE): }, 'skip': 'Changes daily', }, { - # HDS live stream with only relinker URL - 'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews', - 'info_dict': { - 'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc', - 'ext': 'flv', - 'title': 'EuroNews', - }, - 'params': { - 'skip_download': True, - }, - }, { # HLS live stream with ContentItem in og:url 'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html', 'info_dict': { diff --git a/youtube_dl/extractor/raywenderlich.py b/youtube_dlc/extractor/raywenderlich.py index 5411ece21..5411ece21 100644 --- a/youtube_dl/extractor/raywenderlich.py +++ b/youtube_dlc/extractor/raywenderlich.py diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dlc/extractor/rbmaradio.py index ae7413fb5..ae7413fb5 100644 --- a/youtube_dl/extractor/rbmaradio.py +++ b/youtube_dlc/extractor/rbmaradio.py diff --git a/youtube_dl/extractor/rds.py b/youtube_dlc/extractor/rds.py index 8c016a77d..8c016a77d 100644 --- a/youtube_dl/extractor/rds.py +++ b/youtube_dlc/extractor/rds.py diff --git a/youtube_dlc/extractor/redbulltv.py b/youtube_dlc/extractor/redbulltv.py new file mode 100644 index 000000000..3aae79f5d --- /dev/null +++ b/youtube_dlc/extractor/redbulltv.py @@ -0,0 +1,229 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import compat_HTTPError +from ..utils import ( + float_or_none, + ExtractorError, +) + + +class RedBullTVIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?redbull(?:\.tv|\.com(?:/[^/]+)?(?:/tv)?)(?:/events/[^/]+)?/(?:videos?|live|(?:film|episode)s)/(?P<id>AP-\w+)' + _TESTS = [{ + # film + 'url': 'https://www.redbull.tv/video/AP-1Q6XCDTAN1W11', + 'md5': 'fb0445b98aa4394e504b413d98031d1f', + 'info_dict': { + 'id': 'AP-1Q6XCDTAN1W11', + 'ext': 'mp4', + 'title': 'ABC of... WRC - ABC of... S1E6', + 'description': 'md5:5c7ed8f4015c8492ecf64b6ab31e7d31', + 'duration': 1582.04, + }, + }, { + # episode + 'url': 'https://www.redbull.tv/video/AP-1PMHKJFCW1W11', + 'info_dict': { + 'id': 'AP-1PMHKJFCW1W11', + 'ext': 'mp4', + 'title': 'Grime - Hashtags S2E4', + 'description': 'md5:5546aa612958c08a98faaad4abce484d', + 'duration': 904, + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'https://www.redbull.com/int-en/tv/video/AP-1UWHCAR9S1W11/rob-meets-sam-gaze?playlist=playlists::3f81040a-2f31-4832-8e2e-545b1d39d173', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/us-en/videos/AP-1YM9QCYE52111', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/us-en/events/AP-1XV2K61Q51W11/live/AP-1XUJ86FDH1W11', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/films/AP-1ZSMAW8FH2111', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/episodes/AP-1TQWK7XE11W11', + 'only_matching': True, + }] + + def extract_info(self, video_id): + session = self._download_json( + 'https://api.redbull.tv/v3/session', video_id, + note='Downloading access token', query={ + 'category': 'personal_computer', + 'os_family': 'http', + }) + if session.get('code') == 'error': + raise ExtractorError('%s said: %s' % ( + self.IE_NAME, session['message'])) + token = session['token'] + + try: + video = self._download_json( + 'https://api.redbull.tv/v3/products/' + video_id, + video_id, note='Downloading video information', + headers={'Authorization': token} + ) + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404: + error_message = self._parse_json( + e.cause.read().decode(), video_id)['error'] + raise ExtractorError('%s said: %s' % ( + self.IE_NAME, error_message), expected=True) + raise + + title = video['title'].strip() + + formats = self._extract_m3u8_formats( + 'https://dms.redbull.tv/v3/%s/%s/playlist.m3u8' % (video_id, token), + video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls') + self._sort_formats(formats) + + subtitles = {} + for resource in video.get('resources', []): + if resource.startswith('closed_caption_'): + splitted_resource = resource.split('_') + if splitted_resource[2]: + subtitles.setdefault('en', []).append({ + 'url': 'https://resources.redbull.tv/%s/%s' % (video_id, resource), + 'ext': splitted_resource[2], + }) + + subheading = video.get('subheading') + if subheading: + title += ' - %s' % subheading + + return { + 'id': video_id, + 'title': title, + 'description': video.get('long_description') or video.get( + 'short_description'), + 'duration': float_or_none(video.get('duration'), scale=1000), + 'formats': formats, + 'subtitles': subtitles, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + return self.extract_info(video_id) + + +class RedBullEmbedIE(RedBullTVIE): + _VALID_URL = r'https?://(?:www\.)?redbull\.com/embed/(?P<id>rrn:content:[^:]+:[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}:[a-z]{2}-[A-Z]{2,3})' + _TESTS = [{ + # HLS manifest accessible only using assetId + 'url': 'https://www.redbull.com/embed/rrn:content:episode-videos:f3021f4f-3ed4-51ac-915a-11987126e405:en-INT', + 'only_matching': True, + }] + _VIDEO_ESSENSE_TMPL = '''... on %s { + videoEssence { + attributes + } + }''' + + def _real_extract(self, url): + rrn_id = self._match_id(url) + asset_id = self._download_json( + 'https://edge-graphql.crepo-production.redbullaws.com/v1/graphql', + rrn_id, headers={'API-KEY': 'e90a1ff11335423998b100c929ecc866'}, + query={ + 'query': '''{ + resource(id: "%s", enforceGeoBlocking: false) { + %s + %s + } +}''' % (rrn_id, self._VIDEO_ESSENSE_TMPL % 'LiveVideo', self._VIDEO_ESSENSE_TMPL % 'VideoResource'), + })['data']['resource']['videoEssence']['attributes']['assetId'] + return self.extract_info(asset_id) + + +class RedBullTVRrnContentIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?redbull\.com/(?P<region>[a-z]{2,3})-(?P<lang>[a-z]{2})/tv/(?:video|live|film)/(?P<id>rrn:content:[^:]+:[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})' + _TESTS = [{ + 'url': 'https://www.redbull.com/int-en/tv/video/rrn:content:live-videos:e3e6feb4-e95f-50b7-962a-c70f8fd13c73/mens-dh-finals-fort-william', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/tv/video/rrn:content:videos:a36a0f36-ff1b-5db8-a69d-ee11a14bf48b/tn-ts-style?playlist=rrn:content:event-profiles:83f05926-5de8-5389-b5e4-9bb312d715e8:extras', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/tv/film/rrn:content:films:d1f4d00e-4c04-5d19-b510-a805ffa2ab83/follow-me', + 'only_matching': True, + }] + + def _real_extract(self, url): + region, lang, rrn_id = re.search(self._VALID_URL, url).groups() + rrn_id += ':%s-%s' % (lang, region.upper()) + return self.url_result( + 'https://www.redbull.com/embed/' + rrn_id, + RedBullEmbedIE.ie_key(), rrn_id) + + +class RedBullIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?redbull\.com/(?P<region>[a-z]{2,3})-(?P<lang>[a-z]{2})/(?P<type>(?:episode|film|(?:(?:recap|trailer)-)?video)s|live)/(?!AP-|rrn:content:)(?P<id>[^/?#&]+)' + _TESTS = [{ + 'url': 'https://www.redbull.com/int-en/episodes/grime-hashtags-s02-e04', + 'md5': 'db8271a7200d40053a1809ed0dd574ff', + 'info_dict': { + 'id': 'AA-1MT8DQWA91W14', + 'ext': 'mp4', + 'title': 'Grime - Hashtags S2E4', + 'description': 'md5:5546aa612958c08a98faaad4abce484d', + }, + }, { + 'url': 'https://www.redbull.com/int-en/films/kilimanjaro-mountain-of-greatness', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/recap-videos/uci-mountain-bike-world-cup-2017-mens-xco-finals-from-vallnord', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/trailer-videos/kings-of-content', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/videos/tnts-style-red-bull-dance-your-style-s1-e12', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/int-en/live/mens-dh-finals-fort-william', + 'only_matching': True, + }, { + # only available on the int-en website so a fallback is need for the API + # https://www.redbull.com/v3/api/graphql/v1/v3/query/en-GB>en-INT?filter[uriSlug]=fia-wrc-saturday-recap-estonia&rb3Schema=v1:hero + 'url': 'https://www.redbull.com/gb-en/live/fia-wrc-saturday-recap-estonia', + 'only_matching': True, + }] + _INT_FALLBACK_LIST = ['de', 'en', 'es', 'fr'] + _LAT_FALLBACK_MAP = ['ar', 'bo', 'car', 'cl', 'co', 'mx', 'pe'] + + def _real_extract(self, url): + region, lang, filter_type, display_id = re.search(self._VALID_URL, url).groups() + if filter_type == 'episodes': + filter_type = 'episode-videos' + elif filter_type == 'live': + filter_type = 'live-videos' + + regions = [region.upper()] + if region != 'int': + if region in self._LAT_FALLBACK_MAP: + regions.append('LAT') + if lang in self._INT_FALLBACK_LIST: + regions.append('INT') + locale = '>'.join(['%s-%s' % (lang, reg) for reg in regions]) + + rrn_id = self._download_json( + 'https://www.redbull.com/v3/api/graphql/v1/v3/query/' + locale, + display_id, query={ + 'filter[type]': filter_type, + 'filter[uriSlug]': display_id, + 'rb3Schema': 'v1:hero', + })['data']['id'] + + return self.url_result( + 'https://www.redbull.com/embed/' + rrn_id, + RedBullEmbedIE.ie_key(), rrn_id) diff --git a/youtube_dl/extractor/reddit.py b/youtube_dlc/extractor/reddit.py index 663f622b3..663f622b3 100644 --- a/youtube_dl/extractor/reddit.py +++ b/youtube_dlc/extractor/reddit.py diff --git a/youtube_dl/extractor/redtube.py b/youtube_dlc/extractor/redtube.py index 2d2f6a98c..a1ca791ca 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dlc/extractor/redtube.py @@ -15,7 +15,7 @@ from ..utils import ( class RedTubeIE(InfoExtractor): - _VALID_URL = r'https?://(?:(?:www\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P<id>[0-9]+)' + _VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P<id>[0-9]+)' _TESTS = [{ 'url': 'http://www.redtube.com/66418', 'md5': 'fc08071233725f26b8f014dba9590005', @@ -31,6 +31,9 @@ class RedTubeIE(InfoExtractor): }, { 'url': 'http://embed.redtube.com/?bgcolor=000000&id=1443286', 'only_matching': True, + }, { + 'url': 'http://it.redtube.com/66418', + 'only_matching': True, }] @staticmethod diff --git a/youtube_dl/extractor/regiotv.py b/youtube_dlc/extractor/regiotv.py index e250a52f0..e250a52f0 100644 --- a/youtube_dl/extractor/regiotv.py +++ b/youtube_dlc/extractor/regiotv.py diff --git a/youtube_dl/extractor/rentv.py b/youtube_dlc/extractor/rentv.py index 7c8909d95..7c8909d95 100644 --- a/youtube_dl/extractor/rentv.py +++ b/youtube_dlc/extractor/rentv.py diff --git a/youtube_dl/extractor/restudy.py b/youtube_dlc/extractor/restudy.py index d47fb45ca..d47fb45ca 100644 --- a/youtube_dl/extractor/restudy.py +++ b/youtube_dlc/extractor/restudy.py diff --git a/youtube_dl/extractor/reuters.py b/youtube_dlc/extractor/reuters.py index 9dc482d21..9dc482d21 100644 --- a/youtube_dl/extractor/reuters.py +++ b/youtube_dlc/extractor/reuters.py diff --git a/youtube_dl/extractor/reverbnation.py b/youtube_dlc/extractor/reverbnation.py index 4cb99c244..4cb99c244 100644 --- a/youtube_dl/extractor/reverbnation.py +++ b/youtube_dlc/extractor/reverbnation.py diff --git a/youtube_dl/extractor/rice.py b/youtube_dlc/extractor/rice.py index f855719ac..f855719ac 100644 --- a/youtube_dl/extractor/rice.py +++ b/youtube_dlc/extractor/rice.py diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dlc/extractor/rmcdecouverte.py index c3623edcc..c3623edcc 100644 --- a/youtube_dl/extractor/rmcdecouverte.py +++ b/youtube_dlc/extractor/rmcdecouverte.py diff --git a/youtube_dl/extractor/ro220.py b/youtube_dlc/extractor/ro220.py index 69934ef2b..69934ef2b 100644 --- a/youtube_dl/extractor/ro220.py +++ b/youtube_dlc/extractor/ro220.py diff --git a/youtube_dl/extractor/rockstargames.py b/youtube_dlc/extractor/rockstargames.py index cd6904bc9..cd6904bc9 100644 --- a/youtube_dl/extractor/rockstargames.py +++ b/youtube_dlc/extractor/rockstargames.py diff --git a/youtube_dl/extractor/roosterteeth.py b/youtube_dlc/extractor/roosterteeth.py index 8883639b2..8883639b2 100644 --- a/youtube_dl/extractor/roosterteeth.py +++ b/youtube_dlc/extractor/roosterteeth.py diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dlc/extractor/rottentomatoes.py index 14c8e8236..14c8e8236 100644 --- a/youtube_dl/extractor/rottentomatoes.py +++ b/youtube_dlc/extractor/rottentomatoes.py diff --git a/youtube_dl/extractor/roxwel.py b/youtube_dlc/extractor/roxwel.py index 65284643b..65284643b 100644 --- a/youtube_dl/extractor/roxwel.py +++ b/youtube_dlc/extractor/roxwel.py diff --git a/youtube_dl/extractor/rozhlas.py b/youtube_dlc/extractor/rozhlas.py index fccf69401..fccf69401 100644 --- a/youtube_dl/extractor/rozhlas.py +++ b/youtube_dlc/extractor/rozhlas.py diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dlc/extractor/rtbf.py index 3b0f3080b..3b0f3080b 100644 --- a/youtube_dl/extractor/rtbf.py +++ b/youtube_dlc/extractor/rtbf.py diff --git a/youtube_dl/extractor/rte.py b/youtube_dlc/extractor/rte.py index 1fbc72915..1fbc72915 100644 --- a/youtube_dl/extractor/rte.py +++ b/youtube_dlc/extractor/rte.py diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dlc/extractor/rtl2.py index 70f000ca8..70f000ca8 100644 --- a/youtube_dl/extractor/rtl2.py +++ b/youtube_dlc/extractor/rtl2.py diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dlc/extractor/rtlnl.py index fadca8c17..9eaa06f25 100644 --- a/youtube_dl/extractor/rtlnl.py +++ b/youtube_dlc/extractor/rtlnl.py @@ -14,12 +14,27 @@ class RtlNlIE(InfoExtractor): _VALID_URL = r'''(?x) https?://(?:(?:www|static)\.)? (?: - rtlxl\.nl/[^\#]*\#!/[^/]+/| - rtl\.nl/(?:(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html|embed)\b.+?\buuid=|video/) + rtlxl\.nl/(?:[^\#]*\#!|programma)/[^/]+/| + rtl\.nl/(?:(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html|embed)\b.+?\buuid=|video/)| + embed\.rtl\.nl/\#uuid= ) (?P<id>[0-9a-f-]+)''' _TESTS = [{ + # new URL schema + 'url': 'https://www.rtlxl.nl/programma/rtl-nieuws/0bd1384d-d970-3086-98bb-5c104e10c26f', + 'md5': '490428f1187b60d714f34e1f2e3af0b6', + 'info_dict': { + 'id': '0bd1384d-d970-3086-98bb-5c104e10c26f', + 'ext': 'mp4', + 'title': 'RTL Nieuws', + 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e', + 'timestamp': 1593293400, + 'upload_date': '20200627', + 'duration': 661.08, + }, + }, { + # old URL schema 'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/82b1aad1-4a14-3d7b-b554-b0aed1b2c416', 'md5': '473d1946c1fdd050b2c0161a4b13c373', 'info_dict': { @@ -31,6 +46,7 @@ class RtlNlIE(InfoExtractor): 'upload_date': '20160429', 'duration': 1167.96, }, + 'skip': '404', }, { # best format available a3t 'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false', @@ -76,6 +92,10 @@ class RtlNlIE(InfoExtractor): }, { 'url': 'https://static.rtl.nl/embed/?uuid=1a2970fc-5c0b-43ff-9fdc-927e39e6d1bc&autoplay=false&publicatiepunt=rtlnieuwsnl', 'only_matching': True, + }, { + # new embed URL schema + 'url': 'https://embed.rtl.nl/#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false', + 'only_matching': True, }] def _real_extract(self, url): diff --git a/youtube_dl/extractor/rtp.py b/youtube_dlc/extractor/rtp.py index 02986f442..02986f442 100644 --- a/youtube_dl/extractor/rtp.py +++ b/youtube_dlc/extractor/rtp.py diff --git a/youtube_dl/extractor/rts.py b/youtube_dlc/extractor/rts.py index 48f17b828..48f17b828 100644 --- a/youtube_dl/extractor/rts.py +++ b/youtube_dlc/extractor/rts.py diff --git a/youtube_dl/extractor/rtve.py b/youtube_dlc/extractor/rtve.py index ce9db0629..ce9db0629 100644 --- a/youtube_dl/extractor/rtve.py +++ b/youtube_dlc/extractor/rtve.py diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dlc/extractor/rtvnh.py index 6a00f7007..6a00f7007 100644 --- a/youtube_dl/extractor/rtvnh.py +++ b/youtube_dlc/extractor/rtvnh.py diff --git a/youtube_dl/extractor/rtvs.py b/youtube_dlc/extractor/rtvs.py index 6573b260d..6573b260d 100644 --- a/youtube_dl/extractor/rtvs.py +++ b/youtube_dlc/extractor/rtvs.py diff --git a/youtube_dl/extractor/ruhd.py b/youtube_dlc/extractor/ruhd.py index 3c8053a26..3c8053a26 100644 --- a/youtube_dl/extractor/ruhd.py +++ b/youtube_dlc/extractor/ruhd.py diff --git a/youtube_dl/extractor/rutube.py b/youtube_dlc/extractor/rutube.py index 8f54d5675..8f54d5675 100644 --- a/youtube_dl/extractor/rutube.py +++ b/youtube_dlc/extractor/rutube.py diff --git a/youtube_dl/extractor/rutv.py b/youtube_dlc/extractor/rutv.py index d2713c19a..aceb35994 100644 --- a/youtube_dl/extractor/rutv.py +++ b/youtube_dlc/extractor/rutv.py @@ -139,7 +139,7 @@ class RUTVIE(InfoExtractor): is_live = video_type == 'live' json_data = self._download_json( - 'http://player.rutv.ru/iframe/data%s/id/%s' % ('live' if is_live else 'video', video_id), + 'http://player.vgtrk.com/iframe/data%s/id/%s' % ('live' if is_live else 'video', video_id), video_id, 'Downloading JSON') if json_data['errors']: diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dlc/extractor/ruutu.py index f984040aa..f984040aa 100644 --- a/youtube_dl/extractor/ruutu.py +++ b/youtube_dlc/extractor/ruutu.py diff --git a/youtube_dl/extractor/ruv.py b/youtube_dlc/extractor/ruv.py index 8f3cc4095..8f3cc4095 100644 --- a/youtube_dl/extractor/ruv.py +++ b/youtube_dlc/extractor/ruv.py diff --git a/youtube_dl/extractor/safari.py b/youtube_dlc/extractor/safari.py index 2cc665122..2cc665122 100644 --- a/youtube_dl/extractor/safari.py +++ b/youtube_dlc/extractor/safari.py diff --git a/youtube_dl/extractor/sapo.py b/youtube_dlc/extractor/sapo.py index 49a9b313a..49a9b313a 100644 --- a/youtube_dl/extractor/sapo.py +++ b/youtube_dlc/extractor/sapo.py diff --git a/youtube_dl/extractor/savefrom.py b/youtube_dlc/extractor/savefrom.py index 21e44b69a..21e44b69a 100644 --- a/youtube_dl/extractor/savefrom.py +++ b/youtube_dlc/extractor/savefrom.py diff --git a/youtube_dl/extractor/sbs.py b/youtube_dlc/extractor/sbs.py index 0e623ff7b..0e623ff7b 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dlc/extractor/sbs.py diff --git a/youtube_dl/extractor/screencast.py b/youtube_dlc/extractor/screencast.py index 69a0d01f3..69a0d01f3 100644 --- a/youtube_dl/extractor/screencast.py +++ b/youtube_dlc/extractor/screencast.py diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dlc/extractor/screencastomatic.py index b5e76c9af..b5e76c9af 100644 --- a/youtube_dl/extractor/screencastomatic.py +++ b/youtube_dlc/extractor/screencastomatic.py diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dlc/extractor/scrippsnetworks.py index b40b4c4af..b40b4c4af 100644 --- a/youtube_dl/extractor/scrippsnetworks.py +++ b/youtube_dlc/extractor/scrippsnetworks.py diff --git a/youtube_dl/extractor/scte.py b/youtube_dlc/extractor/scte.py index ca1de63b6..ca1de63b6 100644 --- a/youtube_dl/extractor/scte.py +++ b/youtube_dlc/extractor/scte.py diff --git a/youtube_dl/extractor/seeker.py b/youtube_dlc/extractor/seeker.py index 7872dc80d..7872dc80d 100644 --- a/youtube_dl/extractor/seeker.py +++ b/youtube_dlc/extractor/seeker.py diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dlc/extractor/senateisvp.py index db5ef8b57..db5ef8b57 100644 --- a/youtube_dl/extractor/senateisvp.py +++ b/youtube_dlc/extractor/senateisvp.py diff --git a/youtube_dl/extractor/sendtonews.py b/youtube_dlc/extractor/sendtonews.py index 9d9652949..9d9652949 100644 --- a/youtube_dl/extractor/sendtonews.py +++ b/youtube_dlc/extractor/sendtonews.py diff --git a/youtube_dl/extractor/servus.py b/youtube_dlc/extractor/servus.py index 9401bf2cf..9401bf2cf 100644 --- a/youtube_dl/extractor/servus.py +++ b/youtube_dlc/extractor/servus.py diff --git a/youtube_dl/extractor/sevenplus.py b/youtube_dlc/extractor/sevenplus.py index 84568ac69..84568ac69 100644 --- a/youtube_dl/extractor/sevenplus.py +++ b/youtube_dlc/extractor/sevenplus.py diff --git a/youtube_dl/extractor/sexu.py b/youtube_dlc/extractor/sexu.py index 3df51520b..3df51520b 100644 --- a/youtube_dl/extractor/sexu.py +++ b/youtube_dlc/extractor/sexu.py diff --git a/youtube_dl/extractor/seznamzpravy.py b/youtube_dlc/extractor/seznamzpravy.py index 7a1c7e38b..7a1c7e38b 100644 --- a/youtube_dl/extractor/seznamzpravy.py +++ b/youtube_dlc/extractor/seznamzpravy.py diff --git a/youtube_dl/extractor/shahid.py b/youtube_dlc/extractor/shahid.py index 5c2a6206b..5c2a6206b 100644 --- a/youtube_dl/extractor/shahid.py +++ b/youtube_dlc/extractor/shahid.py diff --git a/youtube_dl/extractor/shared.py b/youtube_dlc/extractor/shared.py index 02295d1a4..02295d1a4 100644 --- a/youtube_dl/extractor/shared.py +++ b/youtube_dlc/extractor/shared.py diff --git a/youtube_dl/extractor/showroomlive.py b/youtube_dlc/extractor/showroomlive.py index efd9d561f..efd9d561f 100644 --- a/youtube_dl/extractor/showroomlive.py +++ b/youtube_dlc/extractor/showroomlive.py diff --git a/youtube_dl/extractor/sina.py b/youtube_dlc/extractor/sina.py index 07b766b4a..07b766b4a 100644 --- a/youtube_dl/extractor/sina.py +++ b/youtube_dlc/extractor/sina.py diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dlc/extractor/sixplay.py index 7ec66ecf3..7ec66ecf3 100644 --- a/youtube_dl/extractor/sixplay.py +++ b/youtube_dlc/extractor/sixplay.py diff --git a/youtube_dl/extractor/sky.py b/youtube_dlc/extractor/sky.py index ea30d6e62..ea30d6e62 100644 --- a/youtube_dl/extractor/sky.py +++ b/youtube_dlc/extractor/sky.py diff --git a/youtube_dl/extractor/skylinewebcams.py b/youtube_dlc/extractor/skylinewebcams.py index b7f8ac736..b7f8ac736 100644 --- a/youtube_dl/extractor/skylinewebcams.py +++ b/youtube_dlc/extractor/skylinewebcams.py diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dlc/extractor/skynewsarabia.py index fffc9aa22..fffc9aa22 100644 --- a/youtube_dl/extractor/skynewsarabia.py +++ b/youtube_dlc/extractor/skynewsarabia.py diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dlc/extractor/slideshare.py index e89ebebe7..e89ebebe7 100644 --- a/youtube_dl/extractor/slideshare.py +++ b/youtube_dlc/extractor/slideshare.py diff --git a/youtube_dl/extractor/slideslive.py b/youtube_dlc/extractor/slideslive.py index d9ea76831..d9ea76831 100644 --- a/youtube_dl/extractor/slideslive.py +++ b/youtube_dlc/extractor/slideslive.py diff --git a/youtube_dl/extractor/slutload.py b/youtube_dlc/extractor/slutload.py index 661f9e59d..661f9e59d 100644 --- a/youtube_dl/extractor/slutload.py +++ b/youtube_dlc/extractor/slutload.py diff --git a/youtube_dl/extractor/smotri.py b/youtube_dlc/extractor/smotri.py index 45995f30f..45995f30f 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dlc/extractor/smotri.py diff --git a/youtube_dl/extractor/snotr.py b/youtube_dlc/extractor/snotr.py index f77354748..f77354748 100644 --- a/youtube_dl/extractor/snotr.py +++ b/youtube_dlc/extractor/snotr.py diff --git a/youtube_dl/extractor/sohu.py b/youtube_dlc/extractor/sohu.py index a62ed84f1..76b3cc6b6 100644 --- a/youtube_dl/extractor/sohu.py +++ b/youtube_dlc/extractor/sohu.py @@ -77,7 +77,7 @@ class SohuIE(InfoExtractor): 'info_dict': { 'id': '78932792', 'ext': 'mp4', - 'title': 'youtube-dl testing video', + 'title': 'youtube-dlc testing video', }, 'params': { 'skip_download': True diff --git a/youtube_dl/extractor/sonyliv.py b/youtube_dlc/extractor/sonyliv.py index 58a8c0d4d..58a8c0d4d 100644 --- a/youtube_dl/extractor/sonyliv.py +++ b/youtube_dlc/extractor/sonyliv.py diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dlc/extractor/soundcloud.py index d37c52543..ed70b7169 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dlc/extractor/soundcloud.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import itertools import re +import json +import random from .common import ( InfoExtractor, @@ -28,6 +30,7 @@ from ..utils import ( update_url_query, url_or_none, urlhandle_detect_ext, + sanitized_Request, ) @@ -119,7 +122,7 @@ class SoundcloudIE(InfoExtractor): }, # private link { - 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp', + 'url': 'https://soundcloud.com/jaimemf/youtube-dlc-test-video-a-y-baw/s-8Pjrp', 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604', 'info_dict': { 'id': '123998367', @@ -248,10 +251,15 @@ class SoundcloudIE(InfoExtractor): }, }, { - # with AAC HQ format available via OAuth token + # AAC HQ format available (account with active subscription needed) 'url': 'https://soundcloud.com/wandw/the-chainsmokers-ft-daya-dont-let-me-down-ww-remix-1', 'only_matching': True, }, + { + # Go+ (account with active subscription needed) + 'url': 'https://soundcloud.com/taylorswiftofficial/look-what-you-made-me-do', + 'only_matching': True, + }, ] _API_V2_BASE = 'https://api-v2.soundcloud.com/' @@ -309,7 +317,81 @@ class SoundcloudIE(InfoExtractor): raise def _real_initialize(self): - self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' + self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag" # 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' # 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' + self._login() + + _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + _API_AUTH_QUERY_TEMPLATE = '?client_id=%s' + _API_AUTH_URL_PW = 'https://api-auth.soundcloud.com/web-auth/sign-in/password%s' + _access_token = None + _HEADERS = {} + _NETRC_MACHINE = 'soundcloud' + + def _login(self): + username, password = self._get_login_info() + if username is None: + return + + def genDevId(): + def genNumBlock(): + return ''.join([str(random.randrange(10)) for i in range(6)]) + return '-'.join([genNumBlock() for i in range(4)]) + + payload = { + 'client_id': self._CLIENT_ID, + 'recaptcha_pubkey': 'null', + 'recaptcha_response': 'null', + 'credentials': { + 'identifier': username, + 'password': password + }, + 'signature': self.sign(username, password, self._CLIENT_ID), + 'device_id': genDevId(), + 'user_agent': self._USER_AGENT + } + + query = self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID + login = sanitized_Request(self._API_AUTH_URL_PW % query, json.dumps(payload).encode('utf-8')) + response = self._download_json(login, None) + self._access_token = response.get('session').get('access_token') + if not self._access_token: + self.report_warning('Unable to get access token, login may has failed') + else: + self._HEADERS = {'Authorization': 'OAuth ' + self._access_token} + + # signature generation + def sign(self, user, pw, clid): + a = 33 + i = 1 + s = 440123 + w = 117 + u = 1800000 + l = 1042 + b = 37 + k = 37 + c = 5 + n = "0763ed7314c69015fd4a0dc16bbf4b90" # _KEY + y = "8" # _REV + r = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" # _USER_AGENT + e = user # _USERNAME + t = clid # _CLIENT_ID + + d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) + p = n + y + d + r + e + t + d + n + h = p + + m = 8011470 + f = 0 + + for f in range(f, len(h)): + m = (m >> 1) + ((1 & m) << 23) + m += ord(h[f]) + m &= 16777215 + + # c is not even needed + out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c) + + return out @classmethod def _resolv_url(cls, url): @@ -389,7 +471,7 @@ class SoundcloudIE(InfoExtractor): if not format_url: continue stream = self._download_json( - format_url, track_id, query=query, fatal=False) + format_url, track_id, query=query, fatal=False, headers=self._HEADERS) if not isinstance(stream, dict): continue stream_url = url_or_none(stream.get('url')) @@ -487,7 +569,7 @@ class SoundcloudIE(InfoExtractor): info_json_url = self._resolv_url(self._BASE_URL + resolve_title) info = self._download_json( - info_json_url, full_title, 'Downloading info JSON', query=query) + info_json_url, full_title, 'Downloading info JSON', query=query, headers=self._HEADERS) return self._extract_info_dict(info, full_title, token) @@ -503,7 +585,7 @@ class SoundcloudPlaylistBaseIE(SoundcloudIE): 'ids': ','.join([compat_str(t['id']) for t in tracks]), 'playlistId': playlist_id, 'playlistSecretToken': token, - }) + }, headers=self._HEADERS) entries = [] for track in tracks: track_id = str_or_none(track.get('id')) @@ -523,7 +605,7 @@ class SoundcloudPlaylistBaseIE(SoundcloudIE): class SoundcloudSetIE(SoundcloudPlaylistBaseIE): - _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?' + _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[:\w\d-]+)(?:/(?P<token>[^?/]+))?' IE_NAME = 'soundcloud:set' _TESTS = [{ 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep', @@ -536,6 +618,15 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE): }, { 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token', 'only_matching': True, + }, { + 'url': 'https://soundcloud.com/discover/sets/weekly::flacmatic', + 'only_matching': True, + }, { + 'url': 'https://soundcloud.com/discover/sets/charts-top:all-music:de', + 'only_matching': True, + }, { + 'url': 'https://soundcloud.com/discover/sets/charts-top:hiphoprap:kr', + 'only_matching': True, }] def _real_extract(self, url): @@ -547,7 +638,7 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE): full_title += '/' + token info = self._download_json(self._resolv_url( - self._BASE_URL + full_title), full_title) + self._BASE_URL + full_title), full_title, headers=self._HEADERS) if 'errors' in info: msgs = (compat_str(err['error_message']) for err in info['errors']) @@ -558,8 +649,10 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE): class SoundcloudPagedPlaylistBaseIE(SoundcloudIE): def _extract_playlist(self, base_url, playlist_id, playlist_title): + # Per the SoundCloud documentation, the maximum limit for a linked partioning query is 200. + # https://developers.soundcloud.com/blog/offset-pagination-deprecated COMMON_QUERY = { - 'limit': 80000, + 'limit': 200, 'linked_partitioning': '1', } @@ -572,7 +665,7 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudIE): for i in itertools.count(): response = self._download_json( next_href, playlist_id, - 'Downloading track page %s' % (i + 1), query=query) + 'Downloading track page %s' % (i + 1), query=query, headers=self._HEADERS) collection = response['collection'] @@ -694,7 +787,7 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE): user = self._download_json( self._resolv_url(self._BASE_URL + uploader), - uploader, 'Downloading user info') + uploader, 'Downloading user info', headers=self._HEADERS) resource = mobj.group('rsrc') or 'all' @@ -719,7 +812,7 @@ class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE): def _real_extract(self, url): track_name = self._match_id(url) - track = self._download_json(self._resolv_url(url), track_name) + track = self._download_json(self._resolv_url(url), track_name, headers=self._HEADERS) track_id = self._search_regex( r'soundcloud:track-stations:(\d+)', track['id'], 'track id') @@ -752,7 +845,7 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE): data = self._download_json( self._API_V2_BASE + 'playlists/' + playlist_id, - playlist_id, 'Downloading playlist', query=query) + playlist_id, 'Downloading playlist', query=query, headers=self._HEADERS) return self._extract_set(data, token) @@ -789,7 +882,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE): for i in itertools.count(1): response = self._download_json( next_url, collection_id, 'Downloading page {0}'.format(i), - 'Unable to download API page') + 'Unable to download API page', headers=self._HEADERS) collection = response.get('collection', []) if not collection: diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dlc/extractor/soundgasm.py index 3d78a9d76..3d78a9d76 100644 --- a/youtube_dl/extractor/soundgasm.py +++ b/youtube_dlc/extractor/soundgasm.py diff --git a/youtube_dl/extractor/southpark.py b/youtube_dlc/extractor/southpark.py index da75a43a7..da75a43a7 100644 --- a/youtube_dl/extractor/southpark.py +++ b/youtube_dlc/extractor/southpark.py diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dlc/extractor/spankbang.py index 61ca902ce..61ca902ce 100644 --- a/youtube_dl/extractor/spankbang.py +++ b/youtube_dlc/extractor/spankbang.py diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dlc/extractor/spankwire.py index 35ab9ec37..35ab9ec37 100644 --- a/youtube_dl/extractor/spankwire.py +++ b/youtube_dlc/extractor/spankwire.py diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dlc/extractor/spiegel.py index 4df7f4ddc..4df7f4ddc 100644 --- a/youtube_dl/extractor/spiegel.py +++ b/youtube_dlc/extractor/spiegel.py diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dlc/extractor/spiegeltv.py index 6ccf4c342..6ccf4c342 100644 --- a/youtube_dl/extractor/spiegeltv.py +++ b/youtube_dlc/extractor/spiegeltv.py diff --git a/youtube_dl/extractor/spike.py b/youtube_dlc/extractor/spike.py index aabff7a3c..3cee331f6 100644 --- a/youtube_dl/extractor/spike.py +++ b/youtube_dlc/extractor/spike.py @@ -20,8 +20,18 @@ class BellatorIE(MTVServicesInfoExtractor): _FEED_URL = 'http://www.bellator.com/feeds/mrss/' _GEO_COUNTRIES = ['US'] - def _extract_mgid(self, webpage): - return self._extract_triforce_mgid(webpage) + def _extract_mgid(self, webpage, url): + mgid = None + + if not mgid: + mgid = self._extract_triforce_mgid(webpage) + + if not mgid: + mgid = self._extract_new_triforce_mgid(webpage, url) + + return mgid + +# TODO Remove - Reason: Outdated Site class ParamountNetworkIE(MTVServicesInfoExtractor): @@ -43,7 +53,7 @@ class ParamountNetworkIE(MTVServicesInfoExtractor): _FEED_URL = 'http://www.paramountnetwork.com/feeds/mrss/' _GEO_COUNTRIES = ['US'] - def _extract_mgid(self, webpage): + def _extract_mgid(self, webpage, url): root_data = self._parse_json(self._search_regex( r'window\.__DATA__\s*=\s*({.+})', webpage, 'data'), None) diff --git a/youtube_dl/extractor/sport5.py b/youtube_dlc/extractor/sport5.py index a417b5a4e..a417b5a4e 100644 --- a/youtube_dl/extractor/sport5.py +++ b/youtube_dlc/extractor/sport5.py diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dlc/extractor/sportbox.py index b9017fd2a..b9017fd2a 100644 --- a/youtube_dl/extractor/sportbox.py +++ b/youtube_dlc/extractor/sportbox.py diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dlc/extractor/sportdeutschland.py index 378fc7568..378fc7568 100644 --- a/youtube_dl/extractor/sportdeutschland.py +++ b/youtube_dlc/extractor/sportdeutschland.py diff --git a/youtube_dl/extractor/springboardplatform.py b/youtube_dlc/extractor/springboardplatform.py index 07d99b579..07d99b579 100644 --- a/youtube_dl/extractor/springboardplatform.py +++ b/youtube_dlc/extractor/springboardplatform.py diff --git a/youtube_dl/extractor/sprout.py b/youtube_dlc/extractor/sprout.py index 8467bf49d..8467bf49d 100644 --- a/youtube_dl/extractor/sprout.py +++ b/youtube_dlc/extractor/sprout.py diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dlc/extractor/srgssr.py index 170dce87f..f63a1359a 100644 --- a/youtube_dl/extractor/srgssr.py +++ b/youtube_dlc/extractor/srgssr.py @@ -114,7 +114,7 @@ class SRGSSRPlayIE(InfoExtractor): [^/]+/(?P<type>video|audio)/[^?]+| popup(?P<type_2>video|audio)player ) - \?id=(?P<id>[0-9a-f\-]{36}|\d+) + \?.*?\b(?:id=|urn=urn:[^:]+:video:)(?P<id>[0-9a-f\-]{36}|\d+) ''' _TESTS = [{ @@ -175,6 +175,12 @@ class SRGSSRPlayIE(InfoExtractor): }, { 'url': 'https://www.srf.ch/play/tv/popupvideoplayer?id=c4dba0ca-e75b-43b2-a34f-f708a4932e01', 'only_matching': True, + }, { + 'url': 'https://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?urn=urn:srf:video:28e1a57d-5b76-4399-8ab3-9097f071e6c5', + 'only_matching': True, + }, { + 'url': 'https://www.rts.ch/play/tv/19h30/video/le-19h30?urn=urn:rts:video:6348260', + 'only_matching': True, }] def _real_extract(self, url): diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dlc/extractor/srmediathek.py index 359dadaa3..359dadaa3 100644 --- a/youtube_dl/extractor/srmediathek.py +++ b/youtube_dlc/extractor/srmediathek.py diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dlc/extractor/stanfordoc.py index ae3dd1380..ae3dd1380 100644 --- a/youtube_dl/extractor/stanfordoc.py +++ b/youtube_dlc/extractor/stanfordoc.py diff --git a/youtube_dl/extractor/steam.py b/youtube_dlc/extractor/steam.py index a6a191ceb..a6a191ceb 100644 --- a/youtube_dl/extractor/steam.py +++ b/youtube_dlc/extractor/steam.py diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dlc/extractor/stitcher.py index 97d1ff681..97d1ff681 100644 --- a/youtube_dl/extractor/stitcher.py +++ b/youtube_dlc/extractor/stitcher.py diff --git a/youtube_dlc/extractor/storyfire.py b/youtube_dlc/extractor/storyfire.py new file mode 100644 index 000000000..67457cc94 --- /dev/null +++ b/youtube_dlc/extractor/storyfire.py @@ -0,0 +1,255 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import itertools +from .common import InfoExtractor + + +class StoryFireIE(InfoExtractor): + _VALID_URL = r'(?:(?:https?://(?:www\.)?storyfire\.com/video-details)|(?:https://storyfire.app.link))/(?P<id>[^/\s]+)' + _TESTS = [{ + 'url': 'https://storyfire.com/video-details/5df1d132b6378700117f9181', + 'md5': '560953bfca81a69003cfa5e53ac8a920', + 'info_dict': { + 'id': '5df1d132b6378700117f9181', + 'ext': 'mp4', + 'title': 'Buzzfeed Teaches You About Memes', + 'uploader_id': 'ntZAJFECERSgqHSxzonV5K2E89s1', + 'timestamp': 1576129028, + 'description': 'Mocking Buzzfeed\'s meme lesson. Reuploaded from YouTube because of their new policies', + 'uploader': 'whang!', + 'upload_date': '20191212', + }, + 'params': {'format': 'bestvideo'} # There are no merged formats in the playlist. + }, { + 'url': 'https://storyfire.app.link/5GxAvWOQr8', # Alternate URL format, with unrelated short ID + 'md5': '7a2dc6d60c4889edfed459c620fe690d', + 'info_dict': { + 'id': '5f1e11ecd78a57b6c702001d', + 'ext': 'm4a', + 'title': 'Weird Nintendo Prototype Leaks', + 'description': 'A stream taking a look at some weird Nintendo Prototypes with Luigi in Mario 64 and weird Yoshis', + 'timestamp': 1595808576, + 'upload_date': '20200727', + 'uploader': 'whang!', + 'uploader_id': 'ntZAJFECERSgqHSxzonV5K2E89s1', + }, + 'params': {'format': 'bestaudio'} # Verifying audio extraction + + }] + + _aformats = { + 'audio-medium-audio': {'acodec': 'aac', 'abr': 125, 'preference': -10}, + 'audio-high-audio': {'acodec': 'aac', 'abr': 254, 'preference': -1}, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + # Extracting the json blob is mandatory to proceed with extraction. + jsontext = self._html_search_regex( + r'<script id="__NEXT_DATA__" type="application/json">(.+?)</script>', + webpage, 'json_data') + + json = self._parse_json(jsontext, video_id) + + # The currentVideo field in the json is mandatory + # because it contains the only link to the m3u playlist + video = json['props']['initialState']['video']['currentVideo'] + videourl = video['vimeoVideoURL'] # Video URL is mandatory + + # Extract other fields from the json in an error tolerant fashion + # ID may be incorrect (on short URL format), correct it. + parsed_id = video.get('_id') + if parsed_id: + video_id = parsed_id + + title = video.get('title') + description = video.get('description') + + thumbnail = video.get('storyImage') + views = video.get('views') + likes = video.get('likesCount') + comments = video.get('commentsCount') + duration = video.get('videoDuration') + publishdate = video.get('publishDate') # Apparently epoch time, day only + + uploader = video.get('username') + uploader_id = video.get('hostID') + # Construct an uploader URL + uploader_url = None + if uploader_id: + uploader_url = "https://storyfire.com/user/%s/video" % uploader_id + + # Collect root playlist to determine formats + formats = self._extract_m3u8_formats( + videourl, video_id, 'mp4', 'm3u8_native') + + # Modify formats to fill in missing information about audio codecs + for format in formats: + aformat = self._aformats.get(format['format_id']) + if aformat: + format['acodec'] = aformat['acodec'] + format['abr'] = aformat['abr'] + format['preference'] = aformat['preference'] + format['ext'] = 'm4a' + + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'ext': "mp4", + 'url': videourl, + 'formats': formats, + + 'thumbnail': thumbnail, + 'view_count': views, + 'like_count': likes, + 'comment_count': comments, + 'duration': duration, + 'timestamp': publishdate, + + 'uploader': uploader, + 'uploader_id': uploader_id, + 'uploader_url': uploader_url, + + } + + +class StoryFireUserIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?storyfire\.com/user/(?P<id>[^/\s]+)/video' + _TESTS = [{ + 'url': 'https://storyfire.com/user/ntZAJFECERSgqHSxzonV5K2E89s1/video', + 'info_dict': { + 'id': 'ntZAJFECERSgqHSxzonV5K2E89s1', + 'title': 'whang!', + }, + 'playlist_mincount': 18 + }, { + 'url': 'https://storyfire.com/user/UQ986nFxmAWIgnkZQ0ftVhq4nOk2/video', + 'info_dict': { + 'id': 'UQ986nFxmAWIgnkZQ0ftVhq4nOk2', + 'title': 'McJuggerNuggets', + }, + 'playlist_mincount': 143 + + }] + + # Generator for fetching playlist items + def _enum_videos(self, baseurl, user_id, firstjson): + totalVideos = int(firstjson['videosCount']) + haveVideos = 0 + json = firstjson + + for page in itertools.count(1): + for video in json['videos']: + id = video['_id'] + url = "https://storyfire.com/video-details/%s" % id + haveVideos += 1 + yield { + '_type': 'url', + 'id': id, + 'url': url, + 'ie_key': 'StoryFire', + + 'title': video.get('title'), + 'description': video.get('description'), + 'view_count': video.get('views'), + 'comment_count': video.get('commentsCount'), + 'duration': video.get('videoDuration'), + 'timestamp': video.get('publishDate'), + } + # Are there more pages we could fetch? + if haveVideos < totalVideos: + pageurl = baseurl + ("%i" % haveVideos) + json = self._download_json(pageurl, user_id, + note='Downloading page %s' % page) + + # Are there any videos in the new json? + videos = json.get('videos') + if not videos or len(videos) == 0: + break # no videos + + else: + break # We have fetched all the videos, stop + + def _real_extract(self, url): + user_id = self._match_id(url) + + baseurl = "https://storyfire.com/app/publicVideos/%s?skip=" % user_id + + # Download first page to ensure it can be downloaded, and get user information if available. + firstpage = baseurl + "0" + firstjson = self._download_json(firstpage, user_id) + + title = None + videos = firstjson.get('videos') + if videos and len(videos): + title = videos[1].get('username') + + return { + '_type': 'playlist', + 'entries': self._enum_videos(baseurl, user_id, firstjson), + 'id': user_id, + 'title': title, + } + + +class StoryFireSeriesIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?storyfire\.com/write/series/stories/(?P<id>[^/\s]+)' + _TESTS = [{ + 'url': 'https://storyfire.com/write/series/stories/-Lq6MsuIHLODO6d2dDkr/', + 'info_dict': { + 'id': '-Lq6MsuIHLODO6d2dDkr', + }, + 'playlist_mincount': 13 + }, { + 'url': 'https://storyfire.com/write/series/stories/the_mortal_one/', + 'info_dict': { + 'id': 'the_mortal_one', + }, + 'playlist_count': 0 # This playlist has entries, but no videos. + }, { + 'url': 'https://storyfire.com/write/series/stories/story_time', + 'info_dict': { + 'id': 'story_time', + }, + 'playlist_mincount': 10 + }] + + # Generator for returning playlist items + # This object is substantially different than the one in the user videos page above + def _enum_videos(self, jsonlist): + for video in jsonlist: + id = video['_id'] + if video.get('hasVideo'): # Boolean element + url = "https://storyfire.com/video-details/%s" % id + yield { + '_type': 'url', + 'id': id, + 'url': url, + 'ie_key': 'StoryFire', + + 'title': video.get('title'), + 'description': video.get('description'), + 'view_count': video.get('views'), + 'likes_count': video.get('likesCount'), + 'comment_count': video.get('commentsCount'), + 'duration': video.get('videoDuration'), + 'timestamp': video.get('publishDate'), + } + + def _real_extract(self, url): + list_id = self._match_id(url) + + listurl = "https://storyfire.com/app/seriesStories/%s/list" % list_id + json = self._download_json(listurl, list_id) + + return { + '_type': 'playlist', + 'entries': self._enum_videos(json), + 'id': list_id + } diff --git a/youtube_dl/extractor/streamable.py b/youtube_dlc/extractor/streamable.py index 34725274e..34725274e 100644 --- a/youtube_dl/extractor/streamable.py +++ b/youtube_dlc/extractor/streamable.py diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dlc/extractor/streamcloud.py index b97bb4374..32eb2b92d 100644 --- a/youtube_dl/extractor/streamcloud.py +++ b/youtube_dlc/extractor/streamcloud.py @@ -15,12 +15,12 @@ class StreamcloudIE(InfoExtractor): _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?' _TESTS = [{ - 'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html', + 'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dlc_test_video_____________-BaW_jenozKc.mp4.html', 'md5': '6bea4c7fa5daaacc2a946b7146286686', 'info_dict': { 'id': 'skp9j99s4bpz', 'ext': 'mp4', - 'title': 'youtube-dl test video \'/\\ ä ↭', + 'title': 'youtube-dlc test video \'/\\ ä ↭', }, 'skip': 'Only available from the EU' }, { diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dlc/extractor/streamcz.py index 58e0b4c80..58e0b4c80 100644 --- a/youtube_dl/extractor/streamcz.py +++ b/youtube_dlc/extractor/streamcz.py diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dlc/extractor/streetvoice.py index 91612c7f2..91612c7f2 100644 --- a/youtube_dl/extractor/streetvoice.py +++ b/youtube_dlc/extractor/streetvoice.py diff --git a/youtube_dl/extractor/stretchinternet.py b/youtube_dlc/extractor/stretchinternet.py index 4dbead2ba..4dbead2ba 100644 --- a/youtube_dl/extractor/stretchinternet.py +++ b/youtube_dlc/extractor/stretchinternet.py diff --git a/youtube_dl/extractor/stv.py b/youtube_dlc/extractor/stv.py index bae8b71f4..bae8b71f4 100644 --- a/youtube_dl/extractor/stv.py +++ b/youtube_dlc/extractor/stv.py diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dlc/extractor/sunporno.py index 68051169b..68051169b 100644 --- a/youtube_dl/extractor/sunporno.py +++ b/youtube_dlc/extractor/sunporno.py diff --git a/youtube_dl/extractor/sverigesradio.py b/youtube_dlc/extractor/sverigesradio.py index aa0691f0d..aa0691f0d 100644 --- a/youtube_dl/extractor/sverigesradio.py +++ b/youtube_dlc/extractor/sverigesradio.py diff --git a/youtube_dl/extractor/svt.py b/youtube_dlc/extractor/svt.py index 8e9ec2ca3..2f6887d86 100644 --- a/youtube_dl/extractor/svt.py +++ b/youtube_dlc/extractor/svt.py @@ -231,7 +231,9 @@ class SVTPlayIE(SVTPlayBaseIE): if not svt_id: svt_id = self._search_regex( (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)', - r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"'), + r'["\']videoSvtId["\']\s*:\s*["\']([\da-zA-Z-]+)', + r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"', + r'["\']svtId["\']\s*:\s*["\']([\da-zA-Z-]+)'), webpage, 'video id') return self._extract_by_video_id(svt_id, webpage) diff --git a/youtube_dl/extractor/swrmediathek.py b/youtube_dlc/extractor/swrmediathek.py index 0f615979e..0f615979e 100644 --- a/youtube_dl/extractor/swrmediathek.py +++ b/youtube_dlc/extractor/swrmediathek.py diff --git a/youtube_dl/extractor/syfy.py b/youtube_dlc/extractor/syfy.py index def7e5a2c..def7e5a2c 100644 --- a/youtube_dl/extractor/syfy.py +++ b/youtube_dlc/extractor/syfy.py diff --git a/youtube_dl/extractor/sztvhu.py b/youtube_dlc/extractor/sztvhu.py index cfad33146..cfad33146 100644 --- a/youtube_dl/extractor/sztvhu.py +++ b/youtube_dlc/extractor/sztvhu.py diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dlc/extractor/tagesschau.py index c351b7545..c351b7545 100644 --- a/youtube_dl/extractor/tagesschau.py +++ b/youtube_dlc/extractor/tagesschau.py diff --git a/youtube_dl/extractor/tass.py b/youtube_dlc/extractor/tass.py index 6d336da78..6d336da78 100644 --- a/youtube_dl/extractor/tass.py +++ b/youtube_dlc/extractor/tass.py diff --git a/youtube_dl/extractor/tastytrade.py b/youtube_dlc/extractor/tastytrade.py index 7fe96bd5f..7fe96bd5f 100644 --- a/youtube_dl/extractor/tastytrade.py +++ b/youtube_dlc/extractor/tastytrade.py diff --git a/youtube_dl/extractor/tbs.py b/youtube_dlc/extractor/tbs.py index e8a7c65e0..e8a7c65e0 100644 --- a/youtube_dl/extractor/tbs.py +++ b/youtube_dlc/extractor/tbs.py diff --git a/youtube_dl/extractor/tdslifeway.py b/youtube_dlc/extractor/tdslifeway.py index 101c6ee31..101c6ee31 100644 --- a/youtube_dl/extractor/tdslifeway.py +++ b/youtube_dlc/extractor/tdslifeway.py diff --git a/youtube_dl/extractor/teachable.py b/youtube_dlc/extractor/teachable.py index a75369dbe..a75369dbe 100644 --- a/youtube_dl/extractor/teachable.py +++ b/youtube_dlc/extractor/teachable.py diff --git a/youtube_dl/extractor/teachertube.py b/youtube_dlc/extractor/teachertube.py index 1272078c5..1272078c5 100644 --- a/youtube_dl/extractor/teachertube.py +++ b/youtube_dlc/extractor/teachertube.py diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dlc/extractor/teachingchannel.py index 624cdb3ad..624cdb3ad 100644 --- a/youtube_dl/extractor/teachingchannel.py +++ b/youtube_dlc/extractor/teachingchannel.py diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dlc/extractor/teamcoco.py index 5793b711f..5793b711f 100644 --- a/youtube_dl/extractor/teamcoco.py +++ b/youtube_dlc/extractor/teamcoco.py diff --git a/youtube_dl/extractor/teamtreehouse.py b/youtube_dlc/extractor/teamtreehouse.py index d347e97ef..d347e97ef 100644 --- a/youtube_dl/extractor/teamtreehouse.py +++ b/youtube_dlc/extractor/teamtreehouse.py diff --git a/youtube_dl/extractor/techtalks.py b/youtube_dlc/extractor/techtalks.py index a5b62c717..a5b62c717 100644 --- a/youtube_dl/extractor/techtalks.py +++ b/youtube_dlc/extractor/techtalks.py diff --git a/youtube_dl/extractor/ted.py b/youtube_dlc/extractor/ted.py index 63e2455b2..63e2455b2 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dlc/extractor/ted.py diff --git a/youtube_dl/extractor/tele13.py b/youtube_dlc/extractor/tele13.py index a29a64b6d..a29a64b6d 100644 --- a/youtube_dl/extractor/tele13.py +++ b/youtube_dlc/extractor/tele13.py diff --git a/youtube_dl/extractor/tele5.py b/youtube_dlc/extractor/tele5.py index 3e1a7a9e6..3e1a7a9e6 100644 --- a/youtube_dl/extractor/tele5.py +++ b/youtube_dlc/extractor/tele5.py diff --git a/youtube_dl/extractor/telebruxelles.py b/youtube_dlc/extractor/telebruxelles.py index a0353fe3a..a0353fe3a 100644 --- a/youtube_dl/extractor/telebruxelles.py +++ b/youtube_dlc/extractor/telebruxelles.py diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dlc/extractor/telecinco.py index 9ba3da341..9ba3da341 100644 --- a/youtube_dl/extractor/telecinco.py +++ b/youtube_dlc/extractor/telecinco.py diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dlc/extractor/telegraaf.py index 2dc020537..2dc020537 100644 --- a/youtube_dl/extractor/telegraaf.py +++ b/youtube_dlc/extractor/telegraaf.py diff --git a/youtube_dl/extractor/telemb.py b/youtube_dlc/extractor/telemb.py index 9bcac4ec0..9bcac4ec0 100644 --- a/youtube_dl/extractor/telemb.py +++ b/youtube_dlc/extractor/telemb.py diff --git a/youtube_dl/extractor/telequebec.py b/youtube_dlc/extractor/telequebec.py index c82c94b3a..b4c485b9b 100644 --- a/youtube_dl/extractor/telequebec.py +++ b/youtube_dlc/extractor/telequebec.py @@ -13,14 +13,24 @@ from ..utils import ( class TeleQuebecBaseIE(InfoExtractor): @staticmethod - def _limelight_result(media_id): + def _result(url, ie_key): return { '_type': 'url_transparent', - 'url': smuggle_url( - 'limelight:media:' + media_id, {'geo_countries': ['CA']}), - 'ie_key': 'LimelightMedia', + 'url': smuggle_url(url, {'geo_countries': ['CA']}), + 'ie_key': ie_key, } + @staticmethod + def _limelight_result(media_id): + return TeleQuebecBaseIE._result( + 'limelight:media:' + media_id, 'LimelightMedia') + + @staticmethod + def _brightcove_result(brightcove_id): + return TeleQuebecBaseIE._result( + 'http://players.brightcove.net/6150020952001/default_default/index.html?videoId=%s' + % brightcove_id, 'BrightcoveNew') + class TeleQuebecIE(TeleQuebecBaseIE): _VALID_URL = r'''(?x) @@ -37,11 +47,27 @@ class TeleQuebecIE(TeleQuebecBaseIE): 'id': '577116881b4b439084e6b1cf4ef8b1b3', 'ext': 'mp4', 'title': 'Un petit choc et puis repart!', - 'description': 'md5:b04a7e6b3f74e32d7b294cffe8658374', + 'description': 'md5:067bc84bd6afecad85e69d1000730907', + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'https://zonevideo.telequebec.tv/media/55267/le-soleil/passe-partout', + 'info_dict': { + 'id': '6167180337001', + 'ext': 'mp4', + 'title': 'Le soleil', + 'description': 'md5:64289c922a8de2abbe99c354daffde02', + 'uploader_id': '6150020952001', + 'upload_date': '20200625', + 'timestamp': 1593090307, }, 'params': { + 'format': 'bestvideo', 'skip_download': True, }, + 'add_ie': ['BrightcoveNew'], }, { # no description 'url': 'http://zonevideo.telequebec.tv/media/30261', @@ -58,7 +84,14 @@ class TeleQuebecIE(TeleQuebecBaseIE): 'https://mnmedias.api.telequebec.tv/api/v2/media/' + media_id, media_id)['media'] - info = self._limelight_result(media_data['streamInfo']['sourceId']) + source_id = media_data['streamInfo']['sourceId'] + source = (try_get( + media_data, lambda x: x['streamInfo']['source'], + compat_str) or 'limelight').lower() + if source == 'brightcove': + info = self._brightcove_result(source_id) + else: + info = self._limelight_result(source_id) info.update({ 'title': media_data.get('title'), 'description': try_get( diff --git a/youtube_dl/extractor/teletask.py b/youtube_dlc/extractor/teletask.py index b9e2ef8ca..b9e2ef8ca 100644 --- a/youtube_dl/extractor/teletask.py +++ b/youtube_dlc/extractor/teletask.py diff --git a/youtube_dl/extractor/telewebion.py b/youtube_dlc/extractor/telewebion.py index 1207b1a1b..1207b1a1b 100644 --- a/youtube_dl/extractor/telewebion.py +++ b/youtube_dlc/extractor/telewebion.py diff --git a/youtube_dl/extractor/tennistv.py b/youtube_dlc/extractor/tennistv.py index a586f30ad..a586f30ad 100644 --- a/youtube_dl/extractor/tennistv.py +++ b/youtube_dlc/extractor/tennistv.py diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dlc/extractor/tenplay.py index af325fea8..af325fea8 100644 --- a/youtube_dl/extractor/tenplay.py +++ b/youtube_dlc/extractor/tenplay.py diff --git a/youtube_dl/extractor/testurl.py b/youtube_dlc/extractor/testurl.py index 84a14a0bd..84a14a0bd 100644 --- a/youtube_dl/extractor/testurl.py +++ b/youtube_dlc/extractor/testurl.py diff --git a/youtube_dl/extractor/tf1.py b/youtube_dlc/extractor/tf1.py index 55e2a0721..55e2a0721 100644 --- a/youtube_dl/extractor/tf1.py +++ b/youtube_dlc/extractor/tf1.py diff --git a/youtube_dl/extractor/tfo.py b/youtube_dlc/extractor/tfo.py index 0631cb7ab..0631cb7ab 100644 --- a/youtube_dl/extractor/tfo.py +++ b/youtube_dlc/extractor/tfo.py diff --git a/youtube_dl/extractor/theintercept.py b/youtube_dlc/extractor/theintercept.py index f23b58713..f23b58713 100644 --- a/youtube_dl/extractor/theintercept.py +++ b/youtube_dlc/extractor/theintercept.py diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dlc/extractor/theplatform.py index 07055513a..07055513a 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dlc/extractor/theplatform.py diff --git a/youtube_dl/extractor/thescene.py b/youtube_dlc/extractor/thescene.py index cd642355c..cd642355c 100644 --- a/youtube_dl/extractor/thescene.py +++ b/youtube_dlc/extractor/thescene.py diff --git a/youtube_dl/extractor/thestar.py b/youtube_dlc/extractor/thestar.py index c3f118894..c3f118894 100644 --- a/youtube_dl/extractor/thestar.py +++ b/youtube_dlc/extractor/thestar.py diff --git a/youtube_dl/extractor/thesun.py b/youtube_dlc/extractor/thesun.py index 15d4a6932..15d4a6932 100644 --- a/youtube_dl/extractor/thesun.py +++ b/youtube_dlc/extractor/thesun.py diff --git a/youtube_dl/extractor/theweatherchannel.py b/youtube_dlc/extractor/theweatherchannel.py index c34a49d03..c34a49d03 100644 --- a/youtube_dl/extractor/theweatherchannel.py +++ b/youtube_dlc/extractor/theweatherchannel.py diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dlc/extractor/thisamericanlife.py index 91e45f2c3..91e45f2c3 100644 --- a/youtube_dl/extractor/thisamericanlife.py +++ b/youtube_dlc/extractor/thisamericanlife.py diff --git a/youtube_dl/extractor/thisav.py b/youtube_dlc/extractor/thisav.py index dc3dd03c8..dc3dd03c8 100644 --- a/youtube_dl/extractor/thisav.py +++ b/youtube_dlc/extractor/thisav.py diff --git a/youtube_dl/extractor/thisoldhouse.py b/youtube_dlc/extractor/thisoldhouse.py index a3d9b4017..a3d9b4017 100644 --- a/youtube_dl/extractor/thisoldhouse.py +++ b/youtube_dlc/extractor/thisoldhouse.py diff --git a/youtube_dl/extractor/threeqsdn.py b/youtube_dlc/extractor/threeqsdn.py index f26937da1..f26937da1 100644 --- a/youtube_dl/extractor/threeqsdn.py +++ b/youtube_dlc/extractor/threeqsdn.py diff --git a/youtube_dlc/extractor/tiktok.py b/youtube_dlc/extractor/tiktok.py new file mode 100644 index 000000000..52e5f4f1f --- /dev/null +++ b/youtube_dlc/extractor/tiktok.py @@ -0,0 +1,139 @@ +# coding: utf-8 +from __future__ import unicode_literals +from datetime import datetime + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + int_or_none, + str_or_none, + try_get +) + + +class TikTokBaseIE(InfoExtractor): + def _extract_aweme(self, video_data, webpage, url): + video_info = try_get( + video_data, lambda x: x['videoData']['itemInfos'], dict) + author_info = try_get( + video_data, lambda x: x['videoData']['authorInfos'], dict) + share_info = try_get(video_data, lambda x: x['shareMeta'], dict) + + unique_id = str_or_none(author_info.get('uniqueId')) + timestamp = try_get(video_info, lambda x: int(x['createTime']), int) + date = datetime.fromtimestamp(timestamp).strftime('%Y%m%d') + + height = try_get(video_info, lambda x: x['video']['videoMeta']['height'], int) + width = try_get(video_info, lambda x: x['video']['videoMeta']['width'], int) + thumbnails = [] + thumbnails.append({ + 'url': video_info.get('thumbnail') or self._og_search_thumbnail(webpage), + 'width': width, + 'height': height + }) + + formats = [] + formats.append({ + 'url': try_get(video_info, lambda x: x['video']['urls'][0]), + 'ext': 'mp4', + 'height': height, + 'width': width + }) + + return { + 'comment_count': int_or_none(video_info.get('commentCount')), + 'duration': try_get(video_info, lambda x: x['video']['videoMeta']['duration'], int), + 'height': height, + 'id': str_or_none(video_info.get('id')), + 'like_count': int_or_none(video_info.get('diggCount')), + 'repost_count': int_or_none(video_info.get('shareCount')), + 'thumbnail': try_get(video_info, lambda x: x['covers'][0]), + 'timestamp': timestamp, + 'width': width, + 'title': str_or_none(share_info.get('title')) or self._og_search_title(webpage), + 'creator': str_or_none(author_info.get('nickName')), + 'uploader': unique_id, + 'uploader_id': str_or_none(author_info.get('userId')), + 'uploader_url': 'https://www.tiktok.com/@' + unique_id, + 'thumbnails': thumbnails, + 'upload_date': date, + 'webpage_url': self._og_search_url(webpage), + 'description': str_or_none(video_info.get('text')) or str_or_none(share_info.get('desc')), + 'ext': 'mp4', + 'formats': formats, + 'http_headers': { + 'Referer': url, + } + } + + +class TikTokIE(TikTokBaseIE): + _VALID_URL = r'https?://www\.tiktok\.com/@[\w\._]+/video/(?P<id>\d+)' + + _TESTS = [{ + 'url': 'https://www.tiktok.com/@leenabhushan/video/6748451240264420610', + 'md5': '34a7543afd5a151b0840ba6736fb633b', + 'info_dict': { + 'comment_count': int, + 'creator': 'facestoriesbyleenabh', + 'description': 'md5:a9f6c0c44a1ff2249cae610372d0ae95', + 'duration': 13, + 'ext': 'mp4', + 'formats': list, + 'height': 1280, + 'id': '6748451240264420610', + 'like_count': int, + 'repost_count': int, + 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?', + 'thumbnails': list, + 'timestamp': 1571246252, + 'title': 'facestoriesbyleenabh on TikTok', + 'upload_date': '20191016', + 'uploader': 'leenabhushan', + 'uploader_id': '6691488002098119685', + 'uploader_url': r're:https://www.tiktok.com/@leenabhushan', + 'webpage_url': r're:https://www.tiktok.com/@leenabhushan/(video/)?6748451240264420610', + 'width': 720, + } + }, { + 'url': 'https://www.tiktok.com/@patroxofficial/video/6742501081818877190?langCountry=en', + 'md5': '06b9800d47d5fe51a19e322dd86e61c9', + 'info_dict': { + 'comment_count': int, + 'creator': 'patroX', + 'description': 'md5:5e2a23877420bb85ce6521dbee39ba94', + 'duration': 27, + 'ext': 'mp4', + 'formats': list, + 'height': 960, + 'id': '6742501081818877190', + 'like_count': int, + 'repost_count': int, + 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?', + 'thumbnails': list, + 'timestamp': 1569860870, + 'title': 'patroX on TikTok', + 'upload_date': '20190930', + 'uploader': 'patroxofficial', + 'uploader_id': '18702747', + 'uploader_url': r're:https://www.tiktok.com/@patroxofficial', + 'webpage_url': r're:https://www.tiktok.com/@patroxofficial/(video/)?6742501081818877190', + 'width': 540, + } + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage(url, video_id, note='Downloading video webpage') + json_string = self._search_regex( + r'id=\"__NEXT_DATA__\"\s+type=\"application\/json\"\s*[^>]+>\s*(?P<json_string_ld>[^<]+)', + webpage, 'json_string', group='json_string_ld') + json_data = self._parse_json(json_string, video_id) + video_data = try_get(json_data, lambda x: x['props']['pageProps'], expected_type=dict) + + # Chech statusCode for success + if video_data.get('statusCode') == 0: + return self._extract_aweme(video_data, webpage, url) + + raise ExtractorError('Video not available', video_id=video_id) diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dlc/extractor/tinypic.py index bc2def508..bc2def508 100644 --- a/youtube_dl/extractor/tinypic.py +++ b/youtube_dlc/extractor/tinypic.py diff --git a/youtube_dl/extractor/tmz.py b/youtube_dlc/extractor/tmz.py index 419f9d92e..419f9d92e 100644 --- a/youtube_dl/extractor/tmz.py +++ b/youtube_dlc/extractor/tmz.py diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dlc/extractor/tnaflix.py index b3573c6e0..b3573c6e0 100644 --- a/youtube_dl/extractor/tnaflix.py +++ b/youtube_dlc/extractor/tnaflix.py diff --git a/youtube_dl/extractor/toggle.py b/youtube_dlc/extractor/toggle.py index ca2e36efe..ca2e36efe 100644 --- a/youtube_dl/extractor/toggle.py +++ b/youtube_dlc/extractor/toggle.py diff --git a/youtube_dl/extractor/tonline.py b/youtube_dlc/extractor/tonline.py index cc11eae2a..cc11eae2a 100644 --- a/youtube_dl/extractor/tonline.py +++ b/youtube_dlc/extractor/tonline.py diff --git a/youtube_dl/extractor/toongoggles.py b/youtube_dlc/extractor/toongoggles.py index b5ba1c01d..b5ba1c01d 100644 --- a/youtube_dl/extractor/toongoggles.py +++ b/youtube_dlc/extractor/toongoggles.py diff --git a/youtube_dl/extractor/toutv.py b/youtube_dlc/extractor/toutv.py index 44b022fca..44b022fca 100644 --- a/youtube_dl/extractor/toutv.py +++ b/youtube_dlc/extractor/toutv.py diff --git a/youtube_dl/extractor/toypics.py b/youtube_dlc/extractor/toypics.py index f705a06c9..f705a06c9 100644 --- a/youtube_dl/extractor/toypics.py +++ b/youtube_dlc/extractor/toypics.py diff --git a/youtube_dl/extractor/traileraddict.py b/youtube_dlc/extractor/traileraddict.py index 747370d12..747370d12 100644 --- a/youtube_dl/extractor/traileraddict.py +++ b/youtube_dlc/extractor/traileraddict.py diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dlc/extractor/trilulilu.py index a800449e9..a800449e9 100644 --- a/youtube_dl/extractor/trilulilu.py +++ b/youtube_dlc/extractor/trilulilu.py diff --git a/youtube_dl/extractor/trunews.py b/youtube_dlc/extractor/trunews.py index cca5b5ceb..cca5b5ceb 100644 --- a/youtube_dl/extractor/trunews.py +++ b/youtube_dlc/extractor/trunews.py diff --git a/youtube_dl/extractor/trutv.py b/youtube_dlc/extractor/trutv.py index ce892c8c5..ce892c8c5 100644 --- a/youtube_dl/extractor/trutv.py +++ b/youtube_dlc/extractor/trutv.py diff --git a/youtube_dl/extractor/tube8.py b/youtube_dlc/extractor/tube8.py index db93b0182..db93b0182 100644 --- a/youtube_dl/extractor/tube8.py +++ b/youtube_dlc/extractor/tube8.py diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dlc/extractor/tubitv.py index a51fa6515..a51fa6515 100644 --- a/youtube_dl/extractor/tubitv.py +++ b/youtube_dlc/extractor/tubitv.py diff --git a/youtube_dl/extractor/tudou.py b/youtube_dlc/extractor/tudou.py index 7421378a8..7421378a8 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dlc/extractor/tudou.py diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dlc/extractor/tumblr.py index ae584ad69..ae584ad69 100644 --- a/youtube_dl/extractor/tumblr.py +++ b/youtube_dlc/extractor/tumblr.py diff --git a/youtube_dl/extractor/tunein.py b/youtube_dlc/extractor/tunein.py index c7a5f5a63..c7a5f5a63 100644 --- a/youtube_dl/extractor/tunein.py +++ b/youtube_dlc/extractor/tunein.py diff --git a/youtube_dl/extractor/tunepk.py b/youtube_dlc/extractor/tunepk.py index 9d42651ce..9d42651ce 100644 --- a/youtube_dl/extractor/tunepk.py +++ b/youtube_dlc/extractor/tunepk.py diff --git a/youtube_dl/extractor/turbo.py b/youtube_dlc/extractor/turbo.py index be3eaa5c2..be3eaa5c2 100644 --- a/youtube_dl/extractor/turbo.py +++ b/youtube_dlc/extractor/turbo.py diff --git a/youtube_dl/extractor/turner.py b/youtube_dlc/extractor/turner.py index 4a6cbfbb8..4a6cbfbb8 100644 --- a/youtube_dl/extractor/turner.py +++ b/youtube_dlc/extractor/turner.py diff --git a/youtube_dl/extractor/tv2.py b/youtube_dlc/extractor/tv2.py index 4a19b9be6..4a19b9be6 100644 --- a/youtube_dl/extractor/tv2.py +++ b/youtube_dlc/extractor/tv2.py diff --git a/youtube_dl/extractor/tv2dk.py b/youtube_dlc/extractor/tv2dk.py index 8bda9348d..8bda9348d 100644 --- a/youtube_dl/extractor/tv2dk.py +++ b/youtube_dlc/extractor/tv2dk.py diff --git a/youtube_dl/extractor/tv2hu.py b/youtube_dlc/extractor/tv2hu.py index 86017b757..86017b757 100644 --- a/youtube_dl/extractor/tv2hu.py +++ b/youtube_dlc/extractor/tv2hu.py diff --git a/youtube_dl/extractor/tv4.py b/youtube_dlc/extractor/tv4.py index c498b0191..c498b0191 100644 --- a/youtube_dl/extractor/tv4.py +++ b/youtube_dlc/extractor/tv4.py diff --git a/youtube_dl/extractor/tv5mondeplus.py b/youtube_dlc/extractor/tv5mondeplus.py index b7fe082b9..b7fe082b9 100644 --- a/youtube_dl/extractor/tv5mondeplus.py +++ b/youtube_dlc/extractor/tv5mondeplus.py diff --git a/youtube_dl/extractor/tva.py b/youtube_dlc/extractor/tva.py index 443f46e8a..443f46e8a 100644 --- a/youtube_dl/extractor/tva.py +++ b/youtube_dlc/extractor/tva.py diff --git a/youtube_dl/extractor/tvanouvelles.py b/youtube_dlc/extractor/tvanouvelles.py index 1086176a2..1086176a2 100644 --- a/youtube_dl/extractor/tvanouvelles.py +++ b/youtube_dlc/extractor/tvanouvelles.py diff --git a/youtube_dl/extractor/tvc.py b/youtube_dlc/extractor/tvc.py index 008f64cc2..008f64cc2 100644 --- a/youtube_dl/extractor/tvc.py +++ b/youtube_dlc/extractor/tvc.py diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dlc/extractor/tvigle.py index 180259aba..180259aba 100644 --- a/youtube_dl/extractor/tvigle.py +++ b/youtube_dlc/extractor/tvigle.py diff --git a/youtube_dl/extractor/tvland.py b/youtube_dlc/extractor/tvland.py index 791144128..791144128 100644 --- a/youtube_dl/extractor/tvland.py +++ b/youtube_dlc/extractor/tvland.py diff --git a/youtube_dl/extractor/tvn24.py b/youtube_dlc/extractor/tvn24.py index de0fb5063..de0fb5063 100644 --- a/youtube_dl/extractor/tvn24.py +++ b/youtube_dlc/extractor/tvn24.py diff --git a/youtube_dl/extractor/tvnet.py b/youtube_dlc/extractor/tvnet.py index 4222ff9ee..4222ff9ee 100644 --- a/youtube_dl/extractor/tvnet.py +++ b/youtube_dlc/extractor/tvnet.py diff --git a/youtube_dl/extractor/tvnoe.py b/youtube_dlc/extractor/tvnoe.py index 26a5aeae4..26a5aeae4 100644 --- a/youtube_dl/extractor/tvnoe.py +++ b/youtube_dlc/extractor/tvnoe.py diff --git a/youtube_dl/extractor/tvnow.py b/youtube_dlc/extractor/tvnow.py index 9c8a8a0dc..e2bb62ae8 100644 --- a/youtube_dl/extractor/tvnow.py +++ b/youtube_dlc/extractor/tvnow.py @@ -7,10 +7,12 @@ from .common import InfoExtractor from ..compat import compat_str from ..utils import ( ExtractorError, + get_element_by_id, int_or_none, parse_iso8601, parse_duration, str_or_none, + try_get, update_url_query, urljoin, ) @@ -204,6 +206,86 @@ class TVNowNewIE(InfoExtractor): ie=TVNowIE.ie_key(), video_id=mobj.group('id')) +class TVNowFilmIE(TVNowBaseIE): + _VALID_URL = r'''(?x) + (?P<base_url>https?:// + (?:www\.)?tvnow\.(?:de|at|ch)/ + (?:filme))/ + (?P<title>[^/?$&]+)-(?P<id>\d+) + ''' + _TESTS = [{ + 'url': 'https://www.tvnow.de/filme/lord-of-war-haendler-des-todes-7959', + 'info_dict': { + 'id': '1426690', + 'display_id': 'lord-of-war-haendler-des-todes', + 'ext': 'mp4', + 'title': 'Lord of War', + 'description': 'md5:5eda15c0d5b8cb70dac724c8a0ff89a9', + 'timestamp': 1550010000, + 'upload_date': '20190212', + 'duration': 7016, + }, + }, { + 'url': 'https://www.tvnow.de/filme/the-machinist-12157', + 'info_dict': { + 'id': '328160', + 'display_id': 'the-machinist', + 'ext': 'mp4', + 'title': 'The Machinist', + 'description': 'md5:9a0e363fdd74b3a9e1cdd9e21d0ecc28', + 'timestamp': 1496469720, + 'upload_date': '20170603', + 'duration': 5836, + }, + }, { + 'url': 'https://www.tvnow.de/filme/horst-schlaemmer-isch-kandidiere-17777', + 'only_matching': True, # DRM protected + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + display_id = mobj.group('title') + + webpage = self._download_webpage(url, display_id, fatal=False) + if not webpage: + raise ExtractorError('Cannot download "%s"' % url, expected=True) + + json_text = get_element_by_id('now-web-state', webpage) + if not json_text: + raise ExtractorError('Cannot read video data', expected=True) + + json_data = self._parse_json( + json_text, + display_id, + transform_source=lambda x: x.replace('&q;', '"'), + fatal=False) + if not json_data: + raise ExtractorError('Cannot read video data', expected=True) + + player_key = next( + (key for key in json_data.keys() if 'module/player' in key), + None) + page_key = next( + (key for key in json_data.keys() if 'page/filme' in key), + None) + movie_id = try_get( + json_data, + [ + lambda x: x[player_key]['body']['id'], + lambda x: x[page_key]['body']['modules'][0]['id'], + lambda x: x[page_key]['body']['modules'][1]['id']], + int) + if not movie_id: + raise ExtractorError('Cannot extract movie ID', expected=True) + + info = self._call_api( + 'movies/%d' % movie_id, + display_id, + query={'fields': ','.join(self._VIDEO_FIELDS)}) + + return self._extract_video(info, display_id) + + class TVNowNewBaseIE(InfoExtractor): def _call_api(self, path, video_id, query={}): result = self._download_json( @@ -345,6 +427,82 @@ class TVNowIE(TVNowNewBaseIE): display_id, video_id = re.match(self._VALID_URL, url).groups() info = self._call_api('player/' + video_id, video_id) return self._extract_video(info, video_id, display_id) + + +class TVNowFilmIE(TVNowIE): + _VALID_URL = r'''(?x) + (?P<base_url>https?:// + (?:www\.)?tvnow\.(?:de|at|ch)/ + (?:filme))/ + (?P<title>[^/?$&]+)-(?P<id>\d+) + ''' + _TESTS = [{ + 'url': 'https://www.tvnow.de/filme/lord-of-war-haendler-des-todes-7959', + 'info_dict': { + 'id': '1426690', + 'display_id': 'lord-of-war-haendler-des-todes', + 'ext': 'mp4', + 'title': 'Lord of War', + 'description': 'md5:5eda15c0d5b8cb70dac724c8a0ff89a9', + 'timestamp': 1550010000, + 'upload_date': '20190212', + 'duration': 7016, + }, + }, { + 'url': 'https://www.tvnow.de/filme/the-machinist-12157', + 'info_dict': { + 'id': '328160', + 'display_id': 'the-machinist', + 'ext': 'mp4', + 'title': 'The Machinist', + 'description': 'md5:9a0e363fdd74b3a9e1cdd9e21d0ecc28', + 'timestamp': 1496469720, + 'upload_date': '20170603', + 'duration': 5836, + }, + }, { + 'url': 'https://www.tvnow.de/filme/horst-schlaemmer-isch-kandidiere-17777', + 'only_matching': True, # DRM protected + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + display_id = mobj.group('title') + + webpage = self._download_webpage(url, display_id, fatal=False) + if not webpage: + raise ExtractorError('Cannot download "%s"' % url, expected=True) + + json_text = get_element_by_id('now-web-state', webpage) + if not json_text: + raise ExtractorError('Cannot read video data', expected=True) + + json_data = self._parse_json( + json_text, + display_id, + transform_source=lambda x: x.replace('&q;', '"'), + fatal=False) + if not json_data: + raise ExtractorError('Cannot read video data', expected=True) + + player_key = next( + (key for key in json_data.keys() if 'module/player' in key), + None) + page_key = next( + (key for key in json_data.keys() if 'page/filme' in key), + None) + movie_id = try_get( + json_data, + [ + lambda x: x[player_key]['body']['id'], + lambda x: x[page_key]['body']['modules'][0]['id'], + lambda x: x[page_key]['body']['modules'][1]['id']], + int) + if not movie_id: + raise ExtractorError('Cannot extract movie ID', expected=True) + + info = self._call_api('player/%d' % movie_id, display_id) + return self._extract_video(info, url, display_id) """ diff --git a/youtube_dl/extractor/tvp.py b/youtube_dlc/extractor/tvp.py index accff75b5..accff75b5 100644 --- a/youtube_dl/extractor/tvp.py +++ b/youtube_dlc/extractor/tvp.py diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dlc/extractor/tvplay.py index 3c2450dd0..3c2450dd0 100644 --- a/youtube_dl/extractor/tvplay.py +++ b/youtube_dlc/extractor/tvplay.py diff --git a/youtube_dl/extractor/tvplayer.py b/youtube_dlc/extractor/tvplayer.py index 8f8686a65..8f8686a65 100644 --- a/youtube_dl/extractor/tvplayer.py +++ b/youtube_dlc/extractor/tvplayer.py diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dlc/extractor/tweakers.py index 2b10d9bca..2b10d9bca 100644 --- a/youtube_dl/extractor/tweakers.py +++ b/youtube_dlc/extractor/tweakers.py diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dlc/extractor/twentyfourvideo.py index 74d14049b..74d14049b 100644 --- a/youtube_dl/extractor/twentyfourvideo.py +++ b/youtube_dlc/extractor/twentyfourvideo.py diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dlc/extractor/twentymin.py index a42977f39..a42977f39 100644 --- a/youtube_dl/extractor/twentymin.py +++ b/youtube_dlc/extractor/twentymin.py diff --git a/youtube_dl/extractor/twentythreevideo.py b/youtube_dlc/extractor/twentythreevideo.py index aa0c6e90f..aa0c6e90f 100644 --- a/youtube_dl/extractor/twentythreevideo.py +++ b/youtube_dlc/extractor/twentythreevideo.py diff --git a/youtube_dl/extractor/twitcasting.py b/youtube_dlc/extractor/twitcasting.py index 2dbe89f5b..2dbe89f5b 100644 --- a/youtube_dl/extractor/twitcasting.py +++ b/youtube_dlc/extractor/twitcasting.py diff --git a/youtube_dl/extractor/twitch.py b/youtube_dlc/extractor/twitch.py index eadc48c6d..ab6654432 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dlc/extractor/twitch.py @@ -24,7 +24,6 @@ from ..utils import ( parse_duration, parse_iso8601, qualities, - str_or_none, try_get, unified_timestamp, update_url_query, @@ -337,19 +336,27 @@ def _make_video_result(node): class TwitchGraphQLBaseIE(TwitchBaseIE): _PAGE_LIMIT = 100 - def _download_gql(self, video_id, op, variables, sha256_hash, note, fatal=True): + _OPERATION_HASHES = { + 'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14', + 'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb', + 'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777', + 'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84', + 'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e', + 'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01', + 'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c', + } + + def _download_gql(self, video_id, ops, note, fatal=True): + for op in ops: + op['extensions'] = { + 'persistedQuery': { + 'version': 1, + 'sha256Hash': self._OPERATION_HASHES[op['operationName']], + } + } return self._download_json( 'https://gql.twitch.tv/gql', video_id, note, - data=json.dumps({ - 'operationName': op, - 'variables': variables, - 'extensions': { - 'persistedQuery': { - 'version': 1, - 'sha256Hash': sha256_hash, - } - } - }).encode(), + data=json.dumps(ops).encode(), headers={ 'Content-Type': 'text/plain;charset=UTF-8', 'Client-ID': self._CLIENT_ID, @@ -369,14 +376,15 @@ class TwitchCollectionIE(TwitchGraphQLBaseIE): }] _OPERATION_NAME = 'CollectionSideBar' - _SHA256_HASH = '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14' def _real_extract(self, url): collection_id = self._match_id(url) collection = self._download_gql( - collection_id, self._OPERATION_NAME, - {'collectionID': collection_id}, self._SHA256_HASH, - 'Downloading collection GraphQL')['data']['collection'] + collection_id, [{ + 'operationName': self._OPERATION_NAME, + 'variables': {'collectionID': collection_id}, + }], + 'Downloading collection GraphQL')[0]['data']['collection'] title = collection.get('title') entries = [] for edge in collection['items']['edges']: @@ -403,14 +411,16 @@ class TwitchPlaylistBaseIE(TwitchGraphQLBaseIE): if cursor: variables['cursor'] = cursor page = self._download_gql( - channel_name, self._OPERATION_NAME, variables, - self._SHA256_HASH, + channel_name, [{ + 'operationName': self._OPERATION_NAME, + 'variables': variables, + }], 'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num), fatal=False) if not page: break edges = try_get( - page, lambda x: x['data']['user'][entries_key]['edges'], list) + page, lambda x: x[0]['data']['user'][entries_key]['edges'], list) if not edges: break for edge in edges: @@ -553,7 +563,6 @@ class TwitchVideosIE(TwitchPlaylistBaseIE): 'views': 'Popular', } - _SHA256_HASH = 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb' _OPERATION_NAME = 'FilterableVideoTower_Videos' _ENTRY_KIND = 'video' _EDGE_KIND = 'VideoEdge' @@ -622,7 +631,6 @@ class TwitchVideosClipsIE(TwitchPlaylistBaseIE): # NB: values other than 20 result in skipped videos _PAGE_LIMIT = 20 - _SHA256_HASH = 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777' _OPERATION_NAME = 'ClipsCards__User' _ENTRY_KIND = 'clip' _EDGE_KIND = 'ClipEdge' @@ -680,7 +688,6 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE): 'playlist_mincount': 3, }] - _SHA256_HASH = '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84' _OPERATION_NAME = 'ChannelCollectionsContent' _ENTRY_KIND = 'collection' _EDGE_KIND = 'CollectionsItemEdge' @@ -717,7 +724,7 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE): playlist_title='%s - Collections' % channel_name) -class TwitchStreamIE(TwitchBaseIE): +class TwitchStreamIE(TwitchGraphQLBaseIE): IE_NAME = 'twitch:stream' _VALID_URL = r'''(?x) https?:// @@ -774,28 +781,43 @@ class TwitchStreamIE(TwitchBaseIE): else super(TwitchStreamIE, cls).suitable(url)) def _real_extract(self, url): - channel_name = self._match_id(url) - - access_token = self._download_access_token(channel_name) - - token = access_token['token'] - channel_id = self._extract_channel_id(token, channel_name) + channel_name = self._match_id(url).lower() + + gql = self._download_gql( + channel_name, [{ + 'operationName': 'StreamMetadata', + 'variables': {'channelLogin': channel_name}, + }, { + 'operationName': 'ComscoreStreamingQuery', + 'variables': { + 'channel': channel_name, + 'clipSlug': '', + 'isClip': False, + 'isLive': True, + 'isVodOrCollection': False, + 'vodID': '', + }, + }, { + 'operationName': 'VideoPreviewOverlay', + 'variables': {'login': channel_name}, + }], + 'Downloading stream GraphQL') + + user = gql[0]['data']['user'] + + if not user: + raise ExtractorError( + '%s does not exist' % channel_name, expected=True) - stream = self._call_api( - 'kraken/streams/%s?stream_type=all' % channel_id, - channel_id, 'Downloading stream JSON').get('stream') + stream = user['stream'] if not stream: - raise ExtractorError('%s is offline' % channel_id, expected=True) + raise ExtractorError('%s is offline' % channel_name, expected=True) - # Channel name may be typed if different case than the original channel name - # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing - # an invalid m3u8 URL. Working around by use of original channel name from stream - # JSON and fallback to lowercase if it's not available. - channel_name = try_get( - stream, lambda x: x['channel']['name'], - compat_str) or channel_name.lower() + access_token = self._download_access_token(channel_name) + token = access_token['token'] + stream_id = stream.get('id') or channel_name query = { 'allow_source': 'true', 'allow_audio_only': 'true', @@ -808,41 +830,39 @@ class TwitchStreamIE(TwitchBaseIE): 'token': token.encode('utf-8'), } formats = self._extract_m3u8_formats( - '%s/api/channel/hls/%s.m3u8?%s' - % (self._USHER_BASE, channel_name, compat_urllib_parse_urlencode(query)), - channel_id, 'mp4') + '%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name), + stream_id, 'mp4', query=query) self._prefer_source(formats) view_count = stream.get('viewers') - timestamp = parse_iso8601(stream.get('created_at')) + timestamp = unified_timestamp(stream.get('createdAt')) - channel = stream['channel'] - title = self._live_title(channel.get('display_name') or channel.get('name')) - description = channel.get('status') + sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {} + uploader = sq_user.get('displayName') + description = try_get( + sq_user, lambda x: x['broadcastSettings']['title'], compat_str) - thumbnails = [] - for thumbnail_key, thumbnail_url in stream['preview'].items(): - m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key) - if not m: - continue - thumbnails.append({ - 'url': thumbnail_url, - 'width': int(m.group('width')), - 'height': int(m.group('height')), - }) + thumbnail = url_or_none(try_get( + gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'], + compat_str)) + + title = uploader or channel_name + stream_type = stream.get('type') + if stream_type in ['rerun', 'live']: + title += ' (%s)' % stream_type return { - 'id': str_or_none(stream.get('_id')) or channel_id, + 'id': stream_id, 'display_id': channel_name, - 'title': title, + 'title': self._live_title(title), 'description': description, - 'thumbnails': thumbnails, - 'uploader': channel.get('display_name'), - 'uploader_id': channel.get('name'), + 'thumbnail': thumbnail, + 'uploader': uploader, + 'uploader_id': channel_name, 'timestamp': timestamp, 'view_count': view_count, 'formats': formats, - 'is_live': True, + 'is_live': stream_type == 'live', } diff --git a/youtube_dl/extractor/twitter.py b/youtube_dlc/extractor/twitter.py index 4284487db..4284487db 100644 --- a/youtube_dl/extractor/twitter.py +++ b/youtube_dlc/extractor/twitter.py diff --git a/youtube_dl/extractor/udemy.py b/youtube_dlc/extractor/udemy.py index 2a4faecef..60e364d30 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dlc/extractor/udemy.py @@ -143,7 +143,7 @@ class UdemyIE(InfoExtractor): raise ExtractorError( 'Udemy asks you to solve a CAPTCHA. Login with browser, ' 'solve CAPTCHA, then export cookies and pass cookie file to ' - 'youtube-dl with --cookies.', expected=True) + 'youtube-dlc with --cookies.', expected=True) return ret def _download_json(self, url_or_request, *args, **kwargs): diff --git a/youtube_dl/extractor/udn.py b/youtube_dlc/extractor/udn.py index 2c8e5c7b4..2c8e5c7b4 100644 --- a/youtube_dl/extractor/udn.py +++ b/youtube_dlc/extractor/udn.py diff --git a/youtube_dl/extractor/ufctv.py b/youtube_dlc/extractor/ufctv.py index 3d74ba071..3d74ba071 100644 --- a/youtube_dl/extractor/ufctv.py +++ b/youtube_dlc/extractor/ufctv.py diff --git a/youtube_dl/extractor/uktvplay.py b/youtube_dlc/extractor/uktvplay.py index 2137502a1..2137502a1 100644 --- a/youtube_dl/extractor/uktvplay.py +++ b/youtube_dlc/extractor/uktvplay.py diff --git a/youtube_dl/extractor/umg.py b/youtube_dlc/extractor/umg.py index d815cd9a6..d815cd9a6 100644 --- a/youtube_dl/extractor/umg.py +++ b/youtube_dlc/extractor/umg.py diff --git a/youtube_dl/extractor/unistra.py b/youtube_dlc/extractor/unistra.py index a724cdbef..a724cdbef 100644 --- a/youtube_dl/extractor/unistra.py +++ b/youtube_dlc/extractor/unistra.py diff --git a/youtube_dl/extractor/unity.py b/youtube_dlc/extractor/unity.py index 73daacf29..73daacf29 100644 --- a/youtube_dl/extractor/unity.py +++ b/youtube_dlc/extractor/unity.py diff --git a/youtube_dl/extractor/uol.py b/youtube_dlc/extractor/uol.py index 628adf219..628adf219 100644 --- a/youtube_dl/extractor/uol.py +++ b/youtube_dlc/extractor/uol.py diff --git a/youtube_dl/extractor/uplynk.py b/youtube_dlc/extractor/uplynk.py index f06bf5b12..f06bf5b12 100644 --- a/youtube_dl/extractor/uplynk.py +++ b/youtube_dlc/extractor/uplynk.py diff --git a/youtube_dl/extractor/urort.py b/youtube_dlc/extractor/urort.py index 8f6edab4b..8f6edab4b 100644 --- a/youtube_dl/extractor/urort.py +++ b/youtube_dlc/extractor/urort.py diff --git a/youtube_dl/extractor/urplay.py b/youtube_dlc/extractor/urplay.py index 6030b7cb5..6030b7cb5 100644 --- a/youtube_dl/extractor/urplay.py +++ b/youtube_dlc/extractor/urplay.py diff --git a/youtube_dl/extractor/usanetwork.py b/youtube_dlc/extractor/usanetwork.py index 54c7495cc..54c7495cc 100644 --- a/youtube_dl/extractor/usanetwork.py +++ b/youtube_dlc/extractor/usanetwork.py diff --git a/youtube_dl/extractor/usatoday.py b/youtube_dlc/extractor/usatoday.py index b2103448d..b2103448d 100644 --- a/youtube_dl/extractor/usatoday.py +++ b/youtube_dlc/extractor/usatoday.py diff --git a/youtube_dl/extractor/ustream.py b/youtube_dlc/extractor/ustream.py index 582090d0d..582090d0d 100644 --- a/youtube_dl/extractor/ustream.py +++ b/youtube_dlc/extractor/ustream.py diff --git a/youtube_dl/extractor/ustudio.py b/youtube_dlc/extractor/ustudio.py index 56509beed..56509beed 100644 --- a/youtube_dl/extractor/ustudio.py +++ b/youtube_dlc/extractor/ustudio.py diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dlc/extractor/varzesh3.py index f474ed73f..f474ed73f 100644 --- a/youtube_dl/extractor/varzesh3.py +++ b/youtube_dlc/extractor/varzesh3.py diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dlc/extractor/vbox7.py index 8152acefd..8152acefd 100644 --- a/youtube_dl/extractor/vbox7.py +++ b/youtube_dlc/extractor/vbox7.py diff --git a/youtube_dl/extractor/veehd.py b/youtube_dlc/extractor/veehd.py index a6dc3c8d8..a6dc3c8d8 100644 --- a/youtube_dl/extractor/veehd.py +++ b/youtube_dlc/extractor/veehd.py diff --git a/youtube_dl/extractor/veoh.py b/youtube_dlc/extractor/veoh.py index 1c44c145c..1c44c145c 100644 --- a/youtube_dl/extractor/veoh.py +++ b/youtube_dlc/extractor/veoh.py diff --git a/youtube_dl/extractor/vesti.py b/youtube_dlc/extractor/vesti.py index 5ab716880..5ab716880 100644 --- a/youtube_dl/extractor/vesti.py +++ b/youtube_dlc/extractor/vesti.py diff --git a/youtube_dl/extractor/vevo.py b/youtube_dlc/extractor/vevo.py index 4ea9f1b4b..4ea9f1b4b 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dlc/extractor/vevo.py diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dlc/extractor/vgtv.py index fe7a26b62..fe7a26b62 100644 --- a/youtube_dl/extractor/vgtv.py +++ b/youtube_dlc/extractor/vgtv.py diff --git a/youtube_dl/extractor/vh1.py b/youtube_dlc/extractor/vh1.py index dff94a2b8..ea576dc6b 100644 --- a/youtube_dl/extractor/vh1.py +++ b/youtube_dlc/extractor/vh1.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals from .mtv import MTVServicesInfoExtractor +# TODO Remove - Reason: Outdated Site + class VH1IE(MTVServicesInfoExtractor): IE_NAME = 'vh1.com' diff --git a/youtube_dl/extractor/vice.py b/youtube_dlc/extractor/vice.py index e37499512..e37499512 100644 --- a/youtube_dl/extractor/vice.py +++ b/youtube_dlc/extractor/vice.py diff --git a/youtube_dl/extractor/vidbit.py b/youtube_dlc/extractor/vidbit.py index 91f45b7cc..91f45b7cc 100644 --- a/youtube_dl/extractor/vidbit.py +++ b/youtube_dlc/extractor/vidbit.py diff --git a/youtube_dl/extractor/viddler.py b/youtube_dlc/extractor/viddler.py index 642358433..642358433 100644 --- a/youtube_dl/extractor/viddler.py +++ b/youtube_dlc/extractor/viddler.py diff --git a/youtube_dl/extractor/videa.py b/youtube_dlc/extractor/videa.py index d0e34c819..a03614cc1 100644 --- a/youtube_dl/extractor/videa.py +++ b/youtube_dlc/extractor/videa.py @@ -2,15 +2,24 @@ from __future__ import unicode_literals import re +import random +import string +import struct from .common import InfoExtractor from ..utils import ( + ExtractorError, int_or_none, mimetype2ext, parse_codecs, xpath_element, xpath_text, ) +from ..compat import ( + compat_b64decode, + compat_ord, + compat_parse_qs, +) class VideaIE(InfoExtractor): @@ -60,15 +69,63 @@ class VideaIE(InfoExtractor): r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1', webpage)] + def rc4(self, ciphertext, key): + res = b'' + + keyLen = len(key) + S = list(range(256)) + + j = 0 + for i in range(256): + j = (j + S[i] + ord(key[i % keyLen])) % 256 + S[i], S[j] = S[j], S[i] + + i = 0 + j = 0 + for m in range(len(ciphertext)): + i = (i + 1) % 256 + j = (j + S[i]) % 256 + S[i], S[j] = S[j], S[i] + k = S[(S[i] + S[j]) % 256] + res += struct.pack("B", k ^ compat_ord(ciphertext[m])) + + return res + def _real_extract(self, url): video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id, fatal=True) + error = self._search_regex(r'<p class="error-text">([^<]+)</p>', webpage, 'error', default=None) + if error: + raise ExtractorError(error, expected=True) + + video_src_params_raw = self._search_regex(r'<iframe[^>]+id="videa_player_iframe"[^>]+src="/player\?([^"]+)"', webpage, 'video_src_params') + video_src_params = compat_parse_qs(video_src_params_raw) + player_page = self._download_webpage("https://videa.hu/videojs_player?%s" % video_src_params_raw, video_id, fatal=True) + nonce = self._search_regex(r'_xt\s*=\s*"([^"]+)"', player_page, 'nonce') + random_seed = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(8)) + static_secret = 'xHb0ZvME5q8CBcoQi6AngerDu3FGO9fkUlwPmLVY_RTzj2hJIS4NasXWKy1td7p' + l = nonce[:32] + s = nonce[32:] + result = '' + for i in range(0, 32): + result += s[i - (static_secret.index(l[i]) - 31)] - info = self._download_xml( + video_src_params['_s'] = random_seed + video_src_params['_t'] = result[:16] + encryption_key_stem = result[16:] + random_seed + + [b64_info, handle] = self._download_webpage_handle( 'http://videa.hu/videaplayer_get_xml.php', video_id, - query={'v': video_id}) + query=video_src_params, fatal=True) + + encrypted_info = compat_b64decode(b64_info) + key = encryption_key_stem + handle.info()['x-videa-xs'] + info_str = self.rc4(encrypted_info, key).decode('utf8') + info = self._parse_xml(info_str, video_id) video = xpath_element(info, './/video', 'video', fatal=True) sources = xpath_element(info, './/video_sources', 'sources', fatal=True) + hash_values = xpath_element(info, './/hash_values', 'hash_values', fatal=True) title = xpath_text(video, './title', fatal=True) @@ -77,6 +134,7 @@ class VideaIE(InfoExtractor): source_url = source.text if not source_url: continue + source_url += '?md5=%s&expires=%s' % (hash_values.find('hash_value_%s' % source.get('name')).text, source.get('exp')) f = parse_codecs(source.get('codecs')) f.update({ 'url': source_url, diff --git a/youtube_dl/extractor/videodetective.py b/youtube_dlc/extractor/videodetective.py index fe70db713..fe70db713 100644 --- a/youtube_dl/extractor/videodetective.py +++ b/youtube_dlc/extractor/videodetective.py diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dlc/extractor/videofyme.py index cd3f50a63..cd3f50a63 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dlc/extractor/videofyme.py diff --git a/youtube_dl/extractor/videomore.py b/youtube_dlc/extractor/videomore.py index e3eda3327..e3eda3327 100644 --- a/youtube_dl/extractor/videomore.py +++ b/youtube_dlc/extractor/videomore.py diff --git a/youtube_dl/extractor/videopress.py b/youtube_dlc/extractor/videopress.py index e5f964d39..e5f964d39 100644 --- a/youtube_dl/extractor/videopress.py +++ b/youtube_dlc/extractor/videopress.py diff --git a/youtube_dl/extractor/vidio.py b/youtube_dlc/extractor/vidio.py index b48baf00b..b48baf00b 100644 --- a/youtube_dl/extractor/vidio.py +++ b/youtube_dlc/extractor/vidio.py diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dlc/extractor/vidlii.py index f4774256b..f4774256b 100644 --- a/youtube_dl/extractor/vidlii.py +++ b/youtube_dlc/extractor/vidlii.py diff --git a/youtube_dl/extractor/vidme.py b/youtube_dlc/extractor/vidme.py index 174e69cd6..174e69cd6 100644 --- a/youtube_dl/extractor/vidme.py +++ b/youtube_dlc/extractor/vidme.py diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dlc/extractor/vidzi.py index 42ea4952c..4e79a0b84 100644 --- a/youtube_dl/extractor/vidzi.py +++ b/youtube_dlc/extractor/vidzi.py @@ -20,7 +20,7 @@ class VidziIE(InfoExtractor): 'info_dict': { 'id': 'cghql9yq6emu', 'ext': 'mp4', - 'title': 'youtube-dl test video 1\\\\2\'3/4<5\\\\6ä7↭', + 'title': 'youtube-dlc test video 1\\\\2\'3/4<5\\\\6ä7↭', }, 'params': { # m3u8 download diff --git a/youtube_dl/extractor/vier.py b/youtube_dlc/extractor/vier.py index dbd5ba9ba..dbd5ba9ba 100644 --- a/youtube_dl/extractor/vier.py +++ b/youtube_dlc/extractor/vier.py diff --git a/youtube_dl/extractor/viewlift.py b/youtube_dlc/extractor/viewlift.py index d6b92b1c8..d6b92b1c8 100644 --- a/youtube_dl/extractor/viewlift.py +++ b/youtube_dlc/extractor/viewlift.py diff --git a/youtube_dl/extractor/viidea.py b/youtube_dlc/extractor/viidea.py index a0abbae60..a0abbae60 100644 --- a/youtube_dl/extractor/viidea.py +++ b/youtube_dlc/extractor/viidea.py diff --git a/youtube_dl/extractor/viki.py b/youtube_dlc/extractor/viki.py index b0dcdc0e6..f8e360338 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dlc/extractor/viki.py @@ -12,6 +12,7 @@ from .common import InfoExtractor from ..utils import ( ExtractorError, int_or_none, + HEADRequest, parse_age_limit, parse_iso8601, sanitized_Request, @@ -56,14 +57,14 @@ class VikiBaseIE(InfoExtractor): def _call_api(self, path, video_id, note, timestamp=None, post_data=None): resp = self._download_json( - self._prepare_call(path, timestamp, post_data), video_id, note) + self._prepare_call(path, timestamp, post_data), video_id, note, headers={'x-viki-app-ver': '2.2.5.1428709186'}, expected_status=[200, 400, 404]) error = resp.get('error') if error: if error == 'invalid timestamp': resp = self._download_json( self._prepare_call(path, int(resp['current_timestamp']), post_data), - video_id, '%s (retry)' % note) + video_id, '%s (retry)' % note, headers={'x-viki-app-ver': '2.2.5.1428709186'}, expected_status=[200, 400, 404]) error = resp.get('error') if error: self._raise_error(resp['error']) @@ -220,6 +221,69 @@ class VikiIE(VikiBaseIE): video = self._call_api( 'videos/%s.json' % video_id, video_id, 'Downloading video JSON') + streams = self._call_api( + 'videos/%s/streams.json' % video_id, video_id, + 'Downloading video streams JSON') + + formats = [] + for format_id, stream_dict in streams.items(): + height = int_or_none(self._search_regex( + r'^(\d+)[pP]$', format_id, 'height', default=None)) + for protocol, format_dict in stream_dict.items(): + # rtmps URLs does not seem to work + if protocol == 'rtmps': + continue + format_url = format_dict.get('url') + format_drms = format_dict.get('drms') + format_stream_id = format_dict.get('id') + if format_id == 'm3u8': + m3u8_formats = self._extract_m3u8_formats( + format_url, video_id, 'mp4', + entry_protocol='m3u8_native', + m3u8_id='m3u8-%s' % protocol, fatal=False) + # Despite CODECS metadata in m3u8 all video-only formats + # are actually video+audio + for f in m3u8_formats: + if f.get('acodec') == 'none' and f.get('vcodec') != 'none': + f['acodec'] = None + formats.extend(m3u8_formats) + elif format_id == 'mpd': + mpd_formats = self._extract_mpd_formats( + format_url, video_id, + mpd_id='mpd-%s' % protocol, fatal=False) + formats.extend(mpd_formats) + elif format_id == 'mpd': + + formats.extend(mpd_formats) + elif format_url.startswith('rtmp'): + mobj = re.search( + r'^(?P<url>rtmp://[^/]+/(?P<app>.+?))/(?P<playpath>mp4:.+)$', + format_url) + if not mobj: + continue + formats.append({ + 'format_id': 'rtmp-%s' % format_id, + 'ext': 'flv', + 'url': mobj.group('url'), + 'play_path': mobj.group('playpath'), + 'app': mobj.group('app'), + 'page_url': url, + 'drms': format_drms, + 'stream_id': format_stream_id, + }) + else: + urlh = self._request_webpage( + HEADRequest(format_url), video_id, 'Checking file size', fatal=False) + formats.append({ + 'url': format_url, + 'format_id': '%s-%s' % (format_id, protocol), + 'height': height, + 'drms': format_drms, + 'stream_id': format_stream_id, + 'filesize': int_or_none(urlh.headers.get('Content-Length')), + }) + self._sort_formats(formats) + self._check_errors(video) title = self.dict_selection(video.get('titles', {}), 'en', allow_fallback=False) @@ -244,12 +308,18 @@ class VikiIE(VikiBaseIE): 'url': thumbnail.get('url'), }) + stream_ids = [] + for f in formats: + s_id = f.get('stream_id') + if s_id is not None: + stream_ids.append(s_id) + subtitles = {} for subtitle_lang, _ in video.get('subtitle_completions', {}).items(): subtitles[subtitle_lang] = [{ 'ext': subtitles_format, 'url': self._prepare_call( - 'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)), + 'videos/%s/subtitles/%s.%s?stream_id=%s' % (video_id, subtitle_lang, subtitles_format, stream_ids[0])), } for subtitles_format in ('srt', 'vtt')] result = { @@ -265,10 +335,6 @@ class VikiIE(VikiBaseIE): 'subtitles': subtitles, } - streams = self._call_api( - 'videos/%s/streams.json' % video_id, video_id, - 'Downloading video streams JSON') - if 'external' in streams: result.update({ '_type': 'url_transparent', @@ -276,48 +342,6 @@ class VikiIE(VikiBaseIE): }) return result - formats = [] - for format_id, stream_dict in streams.items(): - height = int_or_none(self._search_regex( - r'^(\d+)[pP]$', format_id, 'height', default=None)) - for protocol, format_dict in stream_dict.items(): - # rtmps URLs does not seem to work - if protocol == 'rtmps': - continue - format_url = format_dict['url'] - if format_id == 'm3u8': - m3u8_formats = self._extract_m3u8_formats( - format_url, video_id, 'mp4', - entry_protocol='m3u8_native', - m3u8_id='m3u8-%s' % protocol, fatal=False) - # Despite CODECS metadata in m3u8 all video-only formats - # are actually video+audio - for f in m3u8_formats: - if f.get('acodec') == 'none' and f.get('vcodec') != 'none': - f['acodec'] = None - formats.extend(m3u8_formats) - elif format_url.startswith('rtmp'): - mobj = re.search( - r'^(?P<url>rtmp://[^/]+/(?P<app>.+?))/(?P<playpath>mp4:.+)$', - format_url) - if not mobj: - continue - formats.append({ - 'format_id': 'rtmp-%s' % format_id, - 'ext': 'flv', - 'url': mobj.group('url'), - 'play_path': mobj.group('playpath'), - 'app': mobj.group('app'), - 'page_url': url, - }) - else: - formats.append({ - 'url': format_url, - 'format_id': '%s-%s' % (format_id, protocol), - 'height': height, - }) - self._sort_formats(formats) - result['formats'] = formats return result diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dlc/extractor/vimeo.py index 421795b94..9839657ca 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dlc/extractor/vimeo.py @@ -600,7 +600,7 @@ class VimeoIE(VimeoBaseInfoExtractor): if b'Because of its privacy settings, this video cannot be played here' in errmsg: raise ExtractorError( 'Cannot download embed-only video without embedding ' - 'URL. Please call youtube-dl with the URL of the page ' + 'URL. Please call youtube-dlc with the URL of the page ' 'that embeds this video.', expected=True) raise diff --git a/youtube_dl/extractor/vimple.py b/youtube_dlc/extractor/vimple.py index c74b43766..c74b43766 100644 --- a/youtube_dl/extractor/vimple.py +++ b/youtube_dlc/extractor/vimple.py diff --git a/youtube_dl/extractor/vine.py b/youtube_dlc/extractor/vine.py index 80b896b56..80b896b56 100644 --- a/youtube_dl/extractor/vine.py +++ b/youtube_dlc/extractor/vine.py diff --git a/youtube_dl/extractor/viqeo.py b/youtube_dlc/extractor/viqeo.py index be7dfa814..be7dfa814 100644 --- a/youtube_dl/extractor/viqeo.py +++ b/youtube_dlc/extractor/viqeo.py diff --git a/youtube_dl/extractor/viu.py b/youtube_dlc/extractor/viu.py index 3bd37525b..3bd37525b 100644 --- a/youtube_dl/extractor/viu.py +++ b/youtube_dlc/extractor/viu.py diff --git a/youtube_dl/extractor/vk.py b/youtube_dlc/extractor/vk.py index 00ec006c4..00ec006c4 100644 --- a/youtube_dl/extractor/vk.py +++ b/youtube_dlc/extractor/vk.py diff --git a/youtube_dl/extractor/vlive.py b/youtube_dlc/extractor/vlive.py index f79531e6f..f79531e6f 100644 --- a/youtube_dl/extractor/vlive.py +++ b/youtube_dlc/extractor/vlive.py diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dlc/extractor/vodlocker.py index 02c9617d2..02c9617d2 100644 --- a/youtube_dl/extractor/vodlocker.py +++ b/youtube_dlc/extractor/vodlocker.py diff --git a/youtube_dl/extractor/vodpl.py b/youtube_dlc/extractor/vodpl.py index 9e919708e..9e919708e 100644 --- a/youtube_dl/extractor/vodpl.py +++ b/youtube_dlc/extractor/vodpl.py diff --git a/youtube_dl/extractor/vodplatform.py b/youtube_dlc/extractor/vodplatform.py index 74d2257e7..74d2257e7 100644 --- a/youtube_dl/extractor/vodplatform.py +++ b/youtube_dlc/extractor/vodplatform.py diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dlc/extractor/voicerepublic.py index a52e40afa..a52e40afa 100644 --- a/youtube_dl/extractor/voicerepublic.py +++ b/youtube_dlc/extractor/voicerepublic.py diff --git a/youtube_dl/extractor/voot.py b/youtube_dlc/extractor/voot.py index 751b21ee5..751b21ee5 100644 --- a/youtube_dl/extractor/voot.py +++ b/youtube_dlc/extractor/voot.py diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dlc/extractor/voxmedia.py index b318e15d4..b318e15d4 100644 --- a/youtube_dl/extractor/voxmedia.py +++ b/youtube_dlc/extractor/voxmedia.py diff --git a/youtube_dl/extractor/vrak.py b/youtube_dlc/extractor/vrak.py index daa247cce..daa247cce 100644 --- a/youtube_dl/extractor/vrak.py +++ b/youtube_dlc/extractor/vrak.py diff --git a/youtube_dl/extractor/vrt.py b/youtube_dlc/extractor/vrt.py index 422025267..2b65d2e5f 100644 --- a/youtube_dl/extractor/vrt.py +++ b/youtube_dlc/extractor/vrt.py @@ -55,13 +55,13 @@ class VRTIE(InfoExtractor): site, display_id = re.match(self._VALID_URL, url).groups() webpage = self._download_webpage(url, display_id) attrs = extract_attributes(self._search_regex( - r'(<[^>]+class="vrtvideo"[^>]*>)', webpage, 'vrt video')) + r'(<[^>]+class="vrtvideo( [^"]*)?"[^>]*>)', webpage, 'vrt video')) - asset_id = attrs['data-videoid'] - publication_id = attrs.get('data-publicationid') + asset_id = attrs['data-video-id'] + publication_id = attrs.get('data-publication-id') if publication_id: asset_id = publication_id + '$' + asset_id - client = attrs.get('data-client') or self._CLIENT_MAP[site] + client = attrs.get('data-client-code') or self._CLIENT_MAP[site] title = strip_or_none(get_element_by_class( 'vrt-title', webpage) or self._html_search_meta( diff --git a/youtube_dl/extractor/vrv.py b/youtube_dlc/extractor/vrv.py index 6e51469b0..6e51469b0 100644 --- a/youtube_dl/extractor/vrv.py +++ b/youtube_dlc/extractor/vrv.py diff --git a/youtube_dl/extractor/vshare.py b/youtube_dlc/extractor/vshare.py index c631ac1fa..c631ac1fa 100644 --- a/youtube_dl/extractor/vshare.py +++ b/youtube_dlc/extractor/vshare.py diff --git a/youtube_dl/extractor/vube.py b/youtube_dlc/extractor/vube.py index 8ce3a6b81..8ce3a6b81 100644 --- a/youtube_dl/extractor/vube.py +++ b/youtube_dlc/extractor/vube.py diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dlc/extractor/vuclip.py index 55e087bdb..55e087bdb 100644 --- a/youtube_dl/extractor/vuclip.py +++ b/youtube_dlc/extractor/vuclip.py diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dlc/extractor/vvvvid.py index 6906cd2ab..6906cd2ab 100644 --- a/youtube_dl/extractor/vvvvid.py +++ b/youtube_dlc/extractor/vvvvid.py diff --git a/youtube_dl/extractor/vyborymos.py b/youtube_dlc/extractor/vyborymos.py index 9e703c4b6..9e703c4b6 100644 --- a/youtube_dl/extractor/vyborymos.py +++ b/youtube_dlc/extractor/vyborymos.py diff --git a/youtube_dl/extractor/vzaar.py b/youtube_dlc/extractor/vzaar.py index b7d02fca3..b7d02fca3 100644 --- a/youtube_dl/extractor/vzaar.py +++ b/youtube_dlc/extractor/vzaar.py diff --git a/youtube_dl/extractor/wakanim.py b/youtube_dlc/extractor/wakanim.py index f9a2395d9..f9a2395d9 100644 --- a/youtube_dl/extractor/wakanim.py +++ b/youtube_dlc/extractor/wakanim.py diff --git a/youtube_dl/extractor/walla.py b/youtube_dlc/extractor/walla.py index cbb548672..cbb548672 100644 --- a/youtube_dl/extractor/walla.py +++ b/youtube_dlc/extractor/walla.py diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dlc/extractor/washingtonpost.py index 625d0a1cc..625d0a1cc 100644 --- a/youtube_dl/extractor/washingtonpost.py +++ b/youtube_dlc/extractor/washingtonpost.py diff --git a/youtube_dl/extractor/wat.py b/youtube_dlc/extractor/wat.py index 8ef3e0906..8ef3e0906 100644 --- a/youtube_dl/extractor/wat.py +++ b/youtube_dlc/extractor/wat.py diff --git a/youtube_dl/extractor/watchbox.py b/youtube_dlc/extractor/watchbox.py index 5a4e46e73..5a4e46e73 100644 --- a/youtube_dl/extractor/watchbox.py +++ b/youtube_dlc/extractor/watchbox.py diff --git a/youtube_dl/extractor/watchindianporn.py b/youtube_dlc/extractor/watchindianporn.py index fadc539ee..fadc539ee 100644 --- a/youtube_dl/extractor/watchindianporn.py +++ b/youtube_dlc/extractor/watchindianporn.py diff --git a/youtube_dl/extractor/wdr.py b/youtube_dlc/extractor/wdr.py index cf6f7c7ed..44d4a13ca 100644 --- a/youtube_dl/extractor/wdr.py +++ b/youtube_dlc/extractor/wdr.py @@ -45,9 +45,18 @@ class WDRIE(InfoExtractor): media_resource = metadata['mediaResource'] formats = [] + subtitles = {} # check if the metadata contains a direct URL to a file for kind, media_resource in media_resource.items(): + if kind == 'captionsHash': + for ext, url in media_resource.items(): + subtitles.setdefault('de', []).append({ + 'url': url, + 'ext': ext, + }) + continue + if kind not in ('dflt', 'alt'): continue @@ -81,14 +90,6 @@ class WDRIE(InfoExtractor): self._sort_formats(formats) - subtitles = {} - caption_url = media_resource.get('captionURL') - if caption_url: - subtitles['de'] = [{ - 'url': caption_url, - 'ext': 'ttml', - }] - title = tracker_data['trackerClipTitle'] return { diff --git a/youtube_dl/extractor/webcaster.py b/youtube_dlc/extractor/webcaster.py index e4b65f54f..e4b65f54f 100644 --- a/youtube_dl/extractor/webcaster.py +++ b/youtube_dlc/extractor/webcaster.py diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dlc/extractor/webofstories.py index f2b8d19b4..f2b8d19b4 100644 --- a/youtube_dl/extractor/webofstories.py +++ b/youtube_dlc/extractor/webofstories.py diff --git a/youtube_dl/extractor/weibo.py b/youtube_dlc/extractor/weibo.py index 621df5b54..621df5b54 100644 --- a/youtube_dl/extractor/weibo.py +++ b/youtube_dlc/extractor/weibo.py diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dlc/extractor/weiqitv.py index 7e0befd39..7e0befd39 100644 --- a/youtube_dl/extractor/weiqitv.py +++ b/youtube_dlc/extractor/weiqitv.py diff --git a/youtube_dl/extractor/wistia.py b/youtube_dlc/extractor/wistia.py index 77febd2eb..77febd2eb 100644 --- a/youtube_dl/extractor/wistia.py +++ b/youtube_dlc/extractor/wistia.py diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dlc/extractor/worldstarhiphop.py index 82587b4ce..82587b4ce 100644 --- a/youtube_dl/extractor/worldstarhiphop.py +++ b/youtube_dlc/extractor/worldstarhiphop.py diff --git a/youtube_dl/extractor/wsj.py b/youtube_dlc/extractor/wsj.py index 67236f377..67236f377 100644 --- a/youtube_dl/extractor/wsj.py +++ b/youtube_dlc/extractor/wsj.py diff --git a/youtube_dl/extractor/wwe.py b/youtube_dlc/extractor/wwe.py index bebc77bb5..bebc77bb5 100644 --- a/youtube_dl/extractor/wwe.py +++ b/youtube_dlc/extractor/wwe.py diff --git a/youtube_dl/extractor/xbef.py b/youtube_dlc/extractor/xbef.py index 4c41e98b2..4c41e98b2 100644 --- a/youtube_dl/extractor/xbef.py +++ b/youtube_dlc/extractor/xbef.py diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dlc/extractor/xboxclips.py index d9c277bc3..d9c277bc3 100644 --- a/youtube_dl/extractor/xboxclips.py +++ b/youtube_dlc/extractor/xboxclips.py diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dlc/extractor/xfileshare.py index 48ef07ed1..48ef07ed1 100644 --- a/youtube_dl/extractor/xfileshare.py +++ b/youtube_dlc/extractor/xfileshare.py diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dlc/extractor/xhamster.py index 76aeaf9a4..76aeaf9a4 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dlc/extractor/xhamster.py diff --git a/youtube_dl/extractor/xiami.py b/youtube_dlc/extractor/xiami.py index 618da8382..618da8382 100644 --- a/youtube_dl/extractor/xiami.py +++ b/youtube_dlc/extractor/xiami.py diff --git a/youtube_dl/extractor/ximalaya.py b/youtube_dlc/extractor/ximalaya.py index a912e54b8..a912e54b8 100644 --- a/youtube_dl/extractor/ximalaya.py +++ b/youtube_dlc/extractor/ximalaya.py diff --git a/youtube_dl/extractor/xminus.py b/youtube_dlc/extractor/xminus.py index 36e5ead1e..36e5ead1e 100644 --- a/youtube_dl/extractor/xminus.py +++ b/youtube_dlc/extractor/xminus.py diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dlc/extractor/xnxx.py index ac1ccc404..ac1ccc404 100644 --- a/youtube_dl/extractor/xnxx.py +++ b/youtube_dlc/extractor/xnxx.py diff --git a/youtube_dl/extractor/xstream.py b/youtube_dlc/extractor/xstream.py index 76c91bd92..76c91bd92 100644 --- a/youtube_dl/extractor/xstream.py +++ b/youtube_dlc/extractor/xstream.py diff --git a/youtube_dl/extractor/xtube.py b/youtube_dlc/extractor/xtube.py index 01b253dcb..01b253dcb 100644 --- a/youtube_dl/extractor/xtube.py +++ b/youtube_dlc/extractor/xtube.py diff --git a/youtube_dl/extractor/xuite.py b/youtube_dlc/extractor/xuite.py index 0276c0dbb..0276c0dbb 100644 --- a/youtube_dl/extractor/xuite.py +++ b/youtube_dlc/extractor/xuite.py diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dlc/extractor/xvideos.py index 8fc64914c..8fc64914c 100644 --- a/youtube_dl/extractor/xvideos.py +++ b/youtube_dlc/extractor/xvideos.py diff --git a/youtube_dl/extractor/xxxymovies.py b/youtube_dlc/extractor/xxxymovies.py index e34ebe3a6..e34ebe3a6 100644 --- a/youtube_dl/extractor/xxxymovies.py +++ b/youtube_dlc/extractor/xxxymovies.py diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dlc/extractor/yahoo.py index e4615376c..e4615376c 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dlc/extractor/yahoo.py diff --git a/youtube_dl/extractor/yandexdisk.py b/youtube_dlc/extractor/yandexdisk.py index e8f6ae10f..e8f6ae10f 100644 --- a/youtube_dl/extractor/yandexdisk.py +++ b/youtube_dlc/extractor/yandexdisk.py diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dlc/extractor/yandexmusic.py index 08d35e04c..4358bc836 100644 --- a/youtube_dl/extractor/yandexmusic.py +++ b/youtube_dlc/extractor/yandexmusic.py @@ -27,12 +27,12 @@ class YandexMusicBaseIE(InfoExtractor): @staticmethod def _raise_captcha(): raise ExtractorError( - 'YandexMusic has considered youtube-dl requests automated and ' + 'YandexMusic has considered youtube-dlc requests automated and ' 'asks you to solve a CAPTCHA. You can either wait for some ' 'time until unblocked and optionally use --sleep-interval ' 'in future or alternatively you can go to https://music.yandex.ru/ ' 'solve CAPTCHA, then export cookies and pass cookie file to ' - 'youtube-dl with --cookies', + 'youtube-dlc with --cookies', expected=True) def _download_webpage_handle(self, *args, **kwargs): diff --git a/youtube_dl/extractor/yandexvideo.py b/youtube_dlc/extractor/yandexvideo.py index 46529be05..46529be05 100644 --- a/youtube_dl/extractor/yandexvideo.py +++ b/youtube_dlc/extractor/yandexvideo.py diff --git a/youtube_dl/extractor/yapfiles.py b/youtube_dlc/extractor/yapfiles.py index cfb368de9..cfb368de9 100644 --- a/youtube_dl/extractor/yapfiles.py +++ b/youtube_dlc/extractor/yapfiles.py diff --git a/youtube_dl/extractor/yesjapan.py b/youtube_dlc/extractor/yesjapan.py index 681338c96..681338c96 100644 --- a/youtube_dl/extractor/yesjapan.py +++ b/youtube_dlc/extractor/yesjapan.py diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dlc/extractor/yinyuetai.py index 1fd8d35c6..1fd8d35c6 100644 --- a/youtube_dl/extractor/yinyuetai.py +++ b/youtube_dlc/extractor/yinyuetai.py diff --git a/youtube_dl/extractor/ynet.py b/youtube_dlc/extractor/ynet.py index c4ae4d88e..c4ae4d88e 100644 --- a/youtube_dl/extractor/ynet.py +++ b/youtube_dlc/extractor/ynet.py diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dlc/extractor/youjizz.py index 88aabd272..88aabd272 100644 --- a/youtube_dl/extractor/youjizz.py +++ b/youtube_dlc/extractor/youjizz.py diff --git a/youtube_dl/extractor/youku.py b/youtube_dlc/extractor/youku.py index 61d1ab209..61d1ab209 100644 --- a/youtube_dl/extractor/youku.py +++ b/youtube_dlc/extractor/youku.py diff --git a/youtube_dl/extractor/younow.py b/youtube_dlc/extractor/younow.py index 04dbc87fc..04dbc87fc 100644 --- a/youtube_dl/extractor/younow.py +++ b/youtube_dlc/extractor/younow.py diff --git a/youtube_dl/extractor/youporn.py b/youtube_dlc/extractor/youporn.py index e7fca22de..e7fca22de 100644 --- a/youtube_dl/extractor/youporn.py +++ b/youtube_dlc/extractor/youporn.py diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dlc/extractor/yourporn.py index 98347491e..98347491e 100644 --- a/youtube_dl/extractor/yourporn.py +++ b/youtube_dlc/extractor/yourporn.py diff --git a/youtube_dl/extractor/yourupload.py b/youtube_dlc/extractor/yourupload.py index 9fa772838..9fa772838 100644 --- a/youtube_dl/extractor/yourupload.py +++ b/youtube_dlc/extractor/yourupload.py diff --git a/youtube_dl/extractor/youtube.py b/youtube_dlc/extractor/youtube.py index 6207585cf..fbfc11563 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dlc/extractor/youtube.py @@ -39,6 +39,7 @@ from ..utils import ( mimetype2ext, orderedSet, parse_codecs, + parse_count, parse_duration, remove_quotes, remove_start, @@ -549,7 +550,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, } - _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt') + _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt') _GEO_BYPASS = False @@ -1264,7 +1265,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'params': { 'skip_download': True, }, - } + }, + { + # empty description results in an empty string + 'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k', + 'info_dict': { + 'id': 'x41yOUIvK2k', + 'ext': 'mp4', + 'title': 'IMG 3456', + 'description': '', + 'upload_date': '20170613', + 'uploader_id': 'ElevageOrVert', + 'uploader': 'ElevageOrVert', + }, + 'params': { + 'skip_download': True, + }, + }, ] def __init__(self, *args, **kwargs): @@ -1435,7 +1452,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): raise ExtractorError( 'Signature extraction failed: ' + tb, cause=e) - def _get_subtitles(self, video_id, webpage): + def _get_subtitles(self, video_id, webpage, has_live_chat_replay): try: subs_doc = self._download_xml( 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id, @@ -1462,6 +1479,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'ext': ext, }) sub_lang_list[lang] = sub_formats + if has_live_chat_replay: + sub_lang_list['live_chat'] = [ + { + 'video_id': video_id, + 'ext': 'json', + 'protocol': 'youtube_live_chat_replay', + }, + ] if not sub_lang_list: self._downloader.report_warning('video doesn\'t have subtitles') return {} @@ -1485,6 +1510,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return self._parse_json( uppercase_escape(config), video_id, fatal=False) + def _get_yt_initial_data(self, video_id, webpage): + config = self._search_regex( + (r'window\["ytInitialData"\]\s*=\s*(.*?)(?<=});', + r'var\s+ytInitialData\s*=\s*(.*?)(?<=});'), + webpage, 'ytInitialData', default=None) + if config: + return self._parse_json( + uppercase_escape(config), video_id, fatal=False) + def _get_automatic_captions(self, video_id, webpage): """We need the webpage for getting the captions url, pass it as an argument to speed up the process.""" @@ -1560,14 +1594,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor): player_response, video_id, fatal=False) if player_response: renderer = player_response['captions']['playerCaptionsTracklistRenderer'] - base_url = renderer['captionTracks'][0]['baseUrl'] - sub_lang_list = [] - for lang in renderer['translationLanguages']: - lang_code = lang.get('languageCode') - if lang_code: - sub_lang_list.append(lang_code) - return make_captions(base_url, sub_lang_list) - + caption_tracks = renderer['captionTracks'] + for caption_track in caption_tracks: + if 'kind' not in caption_track: + # not an automatic transcription + continue + base_url = caption_track['baseUrl'] + sub_lang_list = [] + for lang in renderer['translationLanguages']: + lang_code = lang.get('languageCode') + if lang_code: + sub_lang_list.append(lang_code) + return make_captions(base_url, sub_lang_list) + + self._downloader.report_warning("Couldn't find automatic captions for %s" % video_id) + return {} # Some videos don't provide ttsurl but rather caption_tracks and # caption_translation_languages (e.g. 20LmZk1hakA) # Does not used anymore as of 22.06.2017 @@ -1661,21 +1702,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def _extract_chapters_from_json(self, webpage, video_id, duration): if not webpage: return - player = self._parse_json( + initial_data = self._parse_json( self._search_regex( - r'RELATED_PLAYER_ARGS["\']\s*:\s*({.+})\s*,?\s*\n', webpage, + r'window\["ytInitialData"\] = (.+);\n', webpage, 'player args', default='{}'), video_id, fatal=False) - if not player or not isinstance(player, dict): - return - watch_next_response = player.get('watch_next_response') - if not isinstance(watch_next_response, compat_str): - return - response = self._parse_json(watch_next_response, video_id, fatal=False) - if not response or not isinstance(response, dict): + if not initial_data or not isinstance(initial_data, dict): return chapters_list = try_get( - response, + initial_data, lambda x: x['playerOverlays'] ['playerOverlayRenderer'] ['decoratedPlayerBarRenderer'] @@ -1827,31 +1862,65 @@ class YoutubeIE(YoutubeBaseInfoExtractor): embed_webpage = None if (self._og_search_property('restrictions:age', video_webpage, default=None) == '18+' or re.search(r'player-age-gate-content">', video_webpage) is not None): + cookie_keys = self._get_cookies('https://www.youtube.com').keys() age_gate = True # We simulate the access to the video from www.youtube.com/v/{video_id} # this can be viewed without login into Youtube url = proto + '://www.youtube.com/embed/%s' % video_id embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage') - data = compat_urllib_parse_urlencode({ - 'video_id': video_id, - 'eurl': 'https://youtube.googleapis.com/v/' + video_id, - 'sts': self._search_regex( - r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''), - }) - video_info_url = proto + '://www.youtube.com/get_video_info?' + data - try: - video_info_webpage = self._download_webpage( - video_info_url, video_id, - note='Refetching age-gated info webpage', - errnote='unable to download video info webpage') - except ExtractorError: - video_info_webpage = None - if video_info_webpage: - video_info = compat_parse_qs(video_info_webpage) - pl_response = video_info.get('player_response', [None])[0] - player_response = extract_player_response(pl_response, video_id) - add_dash_mpd(video_info) - view_count = extract_view_count(video_info) + # check if video is only playable on youtube - if so it requires auth (cookies) + if re.search(r'player-unavailable">', embed_webpage) is not None: + ''' + # TODO apply this patch when Support for Python 2.6(!) and above drops + if ({'VISITOR_INFO1_LIVE', 'HSID', 'SSID', 'SID'} <= cookie_keys + or {'VISITOR_INFO1_LIVE', '__Secure-3PSID', 'LOGIN_INFO'} <= cookie_keys): + ''' + if (set(('VISITOR_INFO1_LIVE', 'HSID', 'SSID', 'SID')) <= set(cookie_keys) + or set(('VISITOR_INFO1_LIVE', '__Secure-3PSID', 'LOGIN_INFO')) <= set(cookie_keys)): + age_gate = False + # Try looking directly into the video webpage + ytplayer_config = self._get_ytplayer_config(video_id, video_webpage) + if ytplayer_config: + args = ytplayer_config['args'] + if args.get('url_encoded_fmt_stream_map') or args.get('hlsvp'): + # Convert to the same format returned by compat_parse_qs + video_info = dict((k, [v]) for k, v in args.items()) + add_dash_mpd(video_info) + # Rental video is not rented but preview is available (e.g. + # https://www.youtube.com/watch?v=yYr8q0y5Jfg, + # https://github.com/ytdl-org/youtube-dl/issues/10532) + if not video_info and args.get('ypc_vid'): + return self.url_result( + args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid']) + if args.get('livestream') == '1' or args.get('live_playback') == 1: + is_live = True + if not player_response: + player_response = extract_player_response(args.get('player_response'), video_id) + if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True): + add_dash_mpd_pr(player_response) + else: + raise ExtractorError('Video is age restricted and only playable on Youtube. Requires cookies!', expected=True) + else: + data = compat_urllib_parse_urlencode({ + 'video_id': video_id, + 'eurl': 'https://youtube.googleapis.com/v/' + video_id, + 'sts': self._search_regex( + r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''), + }) + video_info_url = proto + '://www.youtube.com/get_video_info?' + data + try: + video_info_webpage = self._download_webpage( + video_info_url, video_id, + note='Refetching age-gated info webpage', + errnote='unable to download video info webpage') + except ExtractorError: + video_info_webpage = None + if video_info_webpage: + video_info = compat_parse_qs(video_info_webpage) + pl_response = video_info.get('player_response', [None])[0] + player_response = extract_player_response(pl_response, video_id) + add_dash_mpd(video_info) + view_count = extract_view_count(video_info) else: age_gate = False # Try looking directly into the video webpage @@ -1931,7 +2000,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): ''', replace_url, video_description) video_description = clean_html(video_description) else: - video_description = video_details.get('shortDescription') or self._html_search_meta('description', video_webpage) + video_description = video_details.get('shortDescription') + if video_description is None: + video_description = self._html_search_meta('description', video_webpage) if not smuggled_data.get('force_singlefeed', False): if not self._downloader.params.get('noplaylist'): @@ -1985,6 +2056,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if is_live is None: is_live = bool_or_none(video_details.get('isLive')) + has_live_chat_replay = False + if not is_live: + yt_initial_data = self._get_yt_initial_data(video_id, video_webpage) + try: + yt_initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'] + has_live_chat_replay = True + except (KeyError, IndexError, TypeError): + pass + # Check for "rental" videos if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info: raise ExtractorError('"rental" videos not supported. See https://github.com/ytdl-org/youtube-dl/issues/359 for more information.', expected=True) @@ -2199,7 +2279,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): a_format['player_url'] = player_url # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True' - formats.append(a_format) + if self._downloader.params.get('youtube_include_hls_manifest', True): + formats.append(a_format) else: error_message = extract_unavailable_message() if not error_message: @@ -2375,7 +2456,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def _extract_count(count_name): return str_to_int(self._search_regex( - r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>' + r'"accessibilityData":\{"label":"([\d,\w]+) %ss"\}' % re.escape(count_name), video_webpage, count_name, default=None)) @@ -2392,7 +2473,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): or try_get(video_info, lambda x: float_or_none(x['avg_rating'][0]))) # subtitles - video_subtitles = self.extract_subtitles(video_id, video_webpage) + video_subtitles = self.extract_subtitles( + video_id, video_webpage, has_live_chat_replay) automatic_captions = self.extract_automatic_captions(video_id, video_webpage) video_duration = try_get( @@ -2403,6 +2485,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_duration = parse_duration(self._html_search_meta( 'duration', video_webpage, 'video duration')) + # Get Subscriber Count of channel + subscriber_count = parse_count(self._search_regex( + r'"text":"([\d\.]+\w?) subscribers"', + video_webpage, + 'subscriber count', + default=None + )) + # annotations video_annotations = None if self._downloader.params.get('writeannotations', False): @@ -2540,6 +2630,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'album': album, 'release_date': release_date, 'release_year': release_year, + 'subscriber_count': subscriber_count, } diff --git a/youtube_dl/extractor/zapiks.py b/youtube_dlc/extractor/zapiks.py index f6496f516..f6496f516 100644 --- a/youtube_dl/extractor/zapiks.py +++ b/youtube_dlc/extractor/zapiks.py diff --git a/youtube_dl/extractor/zaq1.py b/youtube_dlc/extractor/zaq1.py index 889aff5d8..889aff5d8 100644 --- a/youtube_dl/extractor/zaq1.py +++ b/youtube_dlc/extractor/zaq1.py diff --git a/youtube_dl/extractor/zattoo.py b/youtube_dlc/extractor/zattoo.py index 6bac3026e..6bac3026e 100644 --- a/youtube_dl/extractor/zattoo.py +++ b/youtube_dlc/extractor/zattoo.py diff --git a/youtube_dl/extractor/zdf.py b/youtube_dlc/extractor/zdf.py index 656864b2e..7b5ad4a6e 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dlc/extractor/zdf.py @@ -39,11 +39,23 @@ class ZDFBaseIE(InfoExtractor): class ZDFIE(ZDFBaseIE): - _VALID_URL = r'https?://www\.zdf\.de/(?:[^/]+/)*(?P<id>[^/?]+)\.html' + IE_NAME = "ZDF-3sat" + _VALID_URL = r'https?://www\.(zdf|3sat)\.de/(?:[^/]+/)*(?P<id>[^/?]+)\.html' _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh') _GEO_COUNTRIES = ['DE'] _TESTS = [{ + 'url': 'https://www.3sat.de/wissen/wissenschaftsdoku/luxusgut-lebensraum-100.html', + 'info_dict': { + 'id': 'luxusgut-lebensraum-100', + 'ext': 'mp4', + 'title': 'Luxusgut Lebensraum', + 'description': 'md5:5c09b2f45ac3bc5233d1b50fc543d061', + 'duration': 2601, + 'timestamp': 1566497700, + 'upload_date': '20190822', + } + }, { 'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html', 'info_dict': { 'id': 'die-magie-der-farben-von-koenigspurpur-und-jeansblau-100', diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dlc/extractor/zingmp3.py index adfdcaabf..adfdcaabf 100644 --- a/youtube_dl/extractor/zingmp3.py +++ b/youtube_dlc/extractor/zingmp3.py diff --git a/youtube_dl/extractor/zype.py b/youtube_dlc/extractor/zype.py index 2e2e97a0c..2e2e97a0c 100644 --- a/youtube_dl/extractor/zype.py +++ b/youtube_dlc/extractor/zype.py diff --git a/youtube_dl/jsinterp.py b/youtube_dlc/jsinterp.py index 7bda59610..7bda59610 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dlc/jsinterp.py diff --git a/youtube_dl/options.py b/youtube_dlc/options.py index 6d5ac62b3..1d7a7fed2 100644 --- a/youtube_dl/options.py +++ b/youtube_dlc/options.py @@ -57,33 +57,33 @@ def parseOpts(overrideArguments=None): def _readUserConf(): xdg_config_home = compat_getenv('XDG_CONFIG_HOME') if xdg_config_home: - userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') + userConfFile = os.path.join(xdg_config_home, 'youtube-dlc', 'config') if not os.path.isfile(userConfFile): - userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') + userConfFile = os.path.join(xdg_config_home, 'youtube-dlc.conf') else: - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config') + userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dlc', 'config') if not os.path.isfile(userConfFile): - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf') + userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dlc.conf') userConf = _readOptions(userConfFile, None) if userConf is None: appdata_dir = compat_getenv('appdata') if appdata_dir: userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config'), + os.path.join(appdata_dir, 'youtube-dlc', 'config'), default=None) if userConf is None: userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), + os.path.join(appdata_dir, 'youtube-dlc', 'config.txt'), default=None) if userConf is None: userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf'), + os.path.join(compat_expanduser('~'), 'youtube-dlc.conf'), default=None) if userConf is None: userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'), + os.path.join(compat_expanduser('~'), 'youtube-dlc.conf.txt'), default=None) if userConf is None: @@ -168,14 +168,14 @@ def parseOpts(overrideArguments=None): general.add_option( '--default-search', dest='default_search', metavar='PREFIX', - help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.') + help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dlc "large apple". Use the value "auto" to let youtube-dlc guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.') general.add_option( '--ignore-config', action='store_true', help='Do not read configuration files. ' - 'When given in the global configuration file /etc/youtube-dl.conf: ' - 'Do not read the user configuration in ~/.config/youtube-dl/config ' - '(%APPDATA%/youtube-dl/config.txt on Windows)') + 'When given in the global configuration file /etc/youtube-dlc.conf: ' + 'Do not read the user configuration in ~/.config/youtube-dlc/config ' + '(%APPDATA%/youtube-dlc/config.txt on Windows)') general.add_option( '--config-location', dest='config_location', metavar='PATH', @@ -357,7 +357,7 @@ def parseOpts(overrideArguments=None): authentication.add_option( '-p', '--password', dest='password', metavar='PASSWORD', - help='Account password. If this option is left out, youtube-dl will ask interactively.') + help='Account password. If this option is left out, youtube-dlc will ask interactively.') authentication.add_option( '-2', '--twofactor', dest='twofactor', metavar='TWOFACTOR', @@ -383,7 +383,7 @@ def parseOpts(overrideArguments=None): adobe_pass.add_option( '--ap-password', dest='ap_password', metavar='PASSWORD', - help='Multiple-system operator account password. If this option is left out, youtube-dl will ask interactively.') + help='Multiple-system operator account password. If this option is left out, youtube-dlc will ask interactively.') adobe_pass.add_option( '--ap-list-mso', action='store_true', dest='ap_list_mso', default=False, @@ -415,6 +415,14 @@ def parseOpts(overrideArguments=None): action='store_false', dest='youtube_include_dash_manifest', help='Do not download the DASH manifests and related data on YouTube videos') video_format.add_option( + '--youtube-include-hls-manifest', + action='store_true', dest='youtube_include_hls_manifest', default=True, + help=optparse.SUPPRESS_HELP) + video_format.add_option( + '--youtube-skip-hls-manifest', + action='store_false', dest='youtube_include_hls_manifest', + help='Do not download the HLS manifests and related data on YouTube videos') + video_format.add_option( '--merge-output-format', action='store', dest='merge_output_format', metavar='FORMAT', default=None, help=( @@ -572,6 +580,10 @@ def parseOpts(overrideArguments=None): 'Upper bound of a range for randomized sleep before each download ' '(maximum possible number of seconds to sleep). Must only be used ' 'along with --min-sleep-interval.')) + workarounds.add_option( + '--sleep-subtitles', + dest='sleep_interval_subtitles', action='store_true', default=False, + help='Enforce sleep interval on subtitles as well') verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') verbosity.add_option( @@ -670,11 +682,11 @@ def parseOpts(overrideArguments=None): verbosity.add_option( '-C', '--call-home', dest='call_home', action='store_true', default=False, - help='Contact the youtube-dl server for debugging') + help='Contact the youtube-dlc server for debugging') verbosity.add_option( '--no-call-home', dest='call_home', action='store_false', default=False, - help='Do NOT contact the youtube-dl server for debugging') + help='Do NOT contact the youtube-dlc server for debugging') filesystem = optparse.OptionGroup(parser, 'Filesystem Options') filesystem.add_option( @@ -720,7 +732,7 @@ def parseOpts(overrideArguments=None): filesystem.add_option( '-c', '--continue', action='store_true', dest='continue_dl', default=True, - help='Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.') + help='Force resume of partially downloaded files. By default, youtube-dlc will resume downloads if possible.') filesystem.add_option( '--no-continue', action='store_false', dest='continue_dl', @@ -755,7 +767,7 @@ def parseOpts(overrideArguments=None): help='File to read cookies from and dump cookie jar in') filesystem.add_option( '--cache-dir', dest='cachedir', default=None, metavar='DIR', - help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') + help='Location in the filesystem where youtube-dlc can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dlc or ~/.cache/youtube-dlc . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') filesystem.add_option( '--no-cache-dir', action='store_const', const=False, dest='cachedir', help='Disable filesystem caching') @@ -763,6 +775,9 @@ def parseOpts(overrideArguments=None): '--rm-cache-dir', action='store_true', dest='rm_cachedir', help='Delete all filesystem cache files') + filesystem.add_option( + '--trim-file-name', dest='trim_file_name', default=0, type=int, + help='Limit the filename length (extension excluded)') thumbnail = optparse.OptionGroup(parser, 'Thumbnail images') thumbnail.add_option( @@ -791,6 +806,10 @@ def parseOpts(overrideArguments=None): dest='audioquality', default='5', help='Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)') postproc.add_option( + '--remux-video', + metavar='FORMAT', dest='remuxvideo', default=None, + help='Remux the video to another container format if necessary (currently supported: mp4|mkv, target container format must support video / audio encoding, remuxing may fail)') + postproc.add_option( '--recode-video', metavar='FORMAT', dest='recodevideo', default=None, help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)') @@ -859,6 +878,16 @@ def parseOpts(overrideArguments=None): metavar='FORMAT', dest='convertsubtitles', default=None, help='Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc)') + extractor = optparse.OptionGroup(parser, 'Extractor Options') + extractor.add_option( + '--allow-dynamic-mpd', + action='store_true', dest='dynamic_mpd', default=True, + help=optparse.SUPPRESS_HELP) + extractor.add_option( + '--ignore-dynamic-mpd', + action='store_false', dest='dynamic_mpd', + help='Do not process dynamic DASH manifests') + parser.add_option_group(general) parser.add_option_group(network) parser.add_option_group(geo) @@ -873,6 +902,7 @@ def parseOpts(overrideArguments=None): parser.add_option_group(authentication) parser.add_option_group(adobe_pass) parser.add_option_group(postproc) + parser.add_option_group(extractor) if overrideArguments is not None: opts, args = parser.parse_args(overrideArguments) @@ -892,14 +922,14 @@ def parseOpts(overrideArguments=None): if '--config-location' in command_line_conf: location = compat_expanduser(opts.config_location) if os.path.isdir(location): - location = os.path.join(location, 'youtube-dl.conf') + location = os.path.join(location, 'youtube-dlc.conf') if not os.path.exists(location): parser.error('config-location %s does not exist.' % location) custom_conf = _readOptions(location) elif '--ignore-config' in command_line_conf: pass else: - system_conf = _readOptions('/etc/youtube-dl.conf') + system_conf = _readOptions('/etc/youtube-dlc.conf') if '--ignore-config' not in system_conf: user_conf = _readUserConf() diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dlc/postprocessor/__init__.py index 3ea518399..2c4702823 100644 --- a/youtube_dl/postprocessor/__init__.py +++ b/youtube_dlc/postprocessor/__init__.py @@ -11,6 +11,7 @@ from .ffmpeg import ( FFmpegMergerPP, FFmpegMetadataPP, FFmpegVideoConvertorPP, + FFmpegVideoRemuxerPP, FFmpegSubtitlesConvertorPP, ) from .xattrpp import XAttrMetadataPP @@ -35,6 +36,7 @@ __all__ = [ 'FFmpegPostProcessor', 'FFmpegSubtitlesConvertorPP', 'FFmpegVideoConvertorPP', + 'FFmpegVideoRemuxerPP', 'MetadataFromTitlePP', 'XAttrMetadataPP', ] diff --git a/youtube_dl/postprocessor/common.py b/youtube_dlc/postprocessor/common.py index 599dd1df2..599dd1df2 100644 --- a/youtube_dl/postprocessor/common.py +++ b/youtube_dlc/postprocessor/common.py diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dlc/postprocessor/embedthumbnail.py index 56be914b8..4a0d02fc4 100644 --- a/youtube_dl/postprocessor/embedthumbnail.py +++ b/youtube_dlc/postprocessor/embedthumbnail.py @@ -13,6 +13,7 @@ from ..utils import ( encodeFilename, PostProcessingError, prepend_extension, + replace_extension, shell_quote ) @@ -41,6 +42,38 @@ class EmbedThumbnailPP(FFmpegPostProcessor): 'Skipping embedding the thumbnail because the file is missing.') return [], info + def is_webp(path): + with open(encodeFilename(path), 'rb') as f: + b = f.read(12) + return b[0:4] == b'RIFF' and b[8:] == b'WEBP' + + # Correct extension for WebP file with wrong extension (see #25687, #25717) + _, thumbnail_ext = os.path.splitext(thumbnail_filename) + if thumbnail_ext: + thumbnail_ext = thumbnail_ext[1:].lower() + if thumbnail_ext != 'webp' and is_webp(thumbnail_filename): + self._downloader.to_screen( + '[ffmpeg] Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename) + thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp') + os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename)) + thumbnail_filename = thumbnail_webp_filename + thumbnail_ext = 'webp' + + # Convert unsupported thumbnail formats to JPEG (see #25687, #25717) + if thumbnail_ext not in ['jpg', 'png']: + # NB: % is supposed to be escaped with %% but this does not work + # for input files so working around with standard substitution + escaped_thumbnail_filename = thumbnail_filename.replace('%', '#') + os.rename(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename)) + escaped_thumbnail_jpg_filename = replace_extension(escaped_thumbnail_filename, 'jpg') + self._downloader.to_screen('[ffmpeg] Converting thumbnail "%s" to JPEG' % escaped_thumbnail_filename) + self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_jpg_filename, ['-bsf:v', 'mjpeg2jpeg']) + os.remove(encodeFilename(escaped_thumbnail_filename)) + thumbnail_jpg_filename = replace_extension(thumbnail_filename, 'jpg') + # Rename back to unescaped for further processing + os.rename(encodeFilename(escaped_thumbnail_jpg_filename), encodeFilename(thumbnail_jpg_filename)) + thumbnail_filename = thumbnail_jpg_filename + if info['ext'] == 'mp3': options = [ '-c', 'copy', '-map', '0', '-map', '1', @@ -55,6 +88,25 @@ class EmbedThumbnailPP(FFmpegPostProcessor): os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + elif info['ext'] == 'mkv': + os.rename(encodeFilename(thumbnail_filename), encodeFilename('cover.jpg')) + old_thumbnail_filename = thumbnail_filename + thumbnail_filename = 'cover.jpg' + + options = [ + '-c', 'copy', '-attach', thumbnail_filename, '-metadata:s:t', 'mimetype=image/jpeg'] + + self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename) + + self.run_ffmpeg_multiple_files([filename], temp_filename, options) + + if not self._already_have_thumbnail: + os.remove(encodeFilename(thumbnail_filename)) + else: + os.rename(encodeFilename(thumbnail_filename), encodeFilename(old_thumbnail_filename)) + os.remove(encodeFilename(filename)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + elif info['ext'] in ['m4a', 'mp4']: if not check_executable('AtomicParsley', ['-v']): raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.') diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dlc/postprocessor/execafterdownload.py index 64dabe790..64dabe790 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dlc/postprocessor/execafterdownload.py diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dlc/postprocessor/ffmpeg.py index 5f7298345..5e85f4eeb 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dlc/postprocessor/ffmpeg.py @@ -349,6 +349,27 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): return [path], information +class FFmpegVideoRemuxerPP(FFmpegPostProcessor): + def __init__(self, downloader=None, preferedformat=None): + super(FFmpegVideoRemuxerPP, self).__init__(downloader) + self._preferedformat = preferedformat + + def run(self, information): + path = information['filepath'] + if information['ext'] == self._preferedformat: + self._downloader.to_screen('[ffmpeg] Not remuxing video file %s - already is in target format %s' % (path, self._preferedformat)) + return [], information + options = ['-c', 'copy'] + prefix, sep, ext = path.rpartition('.') + outpath = prefix + sep + self._preferedformat + self._downloader.to_screen('[' + 'ffmpeg' + '] Remuxing video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath) + self.run_ffmpeg(path, outpath, options) + information['filepath'] = outpath + information['format'] = self._preferedformat + information['ext'] = self._preferedformat + return [path], information + + class FFmpegVideoConvertorPP(FFmpegPostProcessor): def __init__(self, downloader=None, preferedformat=None): super(FFmpegVideoConvertorPP, self).__init__(downloader) @@ -476,7 +497,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') in_filenames = [filename] - options = [] + options = ['-map', '0'] if info['ext'] == 'm4a': options.extend(['-vn', '-acodec', 'copy']) @@ -518,7 +539,12 @@ class FFmpegMergerPP(FFmpegPostProcessor): def run(self, info): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') - args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0'] + args = ['-c', 'copy'] + for (i, fmt) in enumerate(info['requested_formats']): + if fmt.get('acodec') != 'none': + args.extend(['-map', '%u:a:0' % (i)]) + if fmt.get('vcodec') != 'none': + args.extend(['-map', '%u:v:0' % (i)]) self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename) self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) @@ -533,7 +559,7 @@ class FFmpegMergerPP(FFmpegPostProcessor): if is_outdated_version( self._versions[self.basename], required_version): warning = ('Your copy of %s is outdated and unable to properly mux separate video and audio files, ' - 'youtube-dl will download single file media. ' + 'youtube-dlc will download single file media. ' 'Update %s to version %s or newer to fix this.') % ( self.basename, self.basename, required_version) if self._downloader: diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dlc/postprocessor/metadatafromtitle.py index f5c14d974..f5c14d974 100644 --- a/youtube_dl/postprocessor/metadatafromtitle.py +++ b/youtube_dlc/postprocessor/metadatafromtitle.py diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dlc/postprocessor/xattrpp.py index 814dabecf..814dabecf 100644 --- a/youtube_dl/postprocessor/xattrpp.py +++ b/youtube_dlc/postprocessor/xattrpp.py diff --git a/youtube_dl/socks.py b/youtube_dlc/socks.py index 5d4adbe72..5d4adbe72 100644 --- a/youtube_dl/socks.py +++ b/youtube_dlc/socks.py diff --git a/youtube_dl/swfinterp.py b/youtube_dlc/swfinterp.py index 0c7158575..0c7158575 100644 --- a/youtube_dl/swfinterp.py +++ b/youtube_dlc/swfinterp.py diff --git a/youtube_dl/update.py b/youtube_dlc/update.py index 84c964617..e49e09c17 100644 --- a/youtube_dl/update.py +++ b/youtube_dlc/update.py @@ -32,13 +32,13 @@ def rsa_verify(message, signature, key): def update_self(to_screen, verbose, opener): """Update the program file with the latest version from the repository""" - UPDATE_URL = 'https://yt-dl.org/update/' + UPDATE_URL = 'https://blackjack4494.github.io//update/' VERSION_URL = UPDATE_URL + 'LATEST_VERSION' JSON_URL = UPDATE_URL + 'versions.json' UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537) if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, 'frozen'): - to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.') + to_screen('It looks like you installed youtube-dlc with a package manager, pip, setup.py or a tarball. Please use that to update.') return # Check if there is a new version @@ -50,7 +50,7 @@ def update_self(to_screen, verbose, opener): to_screen('ERROR: can\'t find the current version. Please try again later.') return if newversion == __version__: - to_screen('youtube-dl is up-to-date (' + __version__ + ')') + to_screen('youtube-dlc is up-to-date (' + __version__ + ')') return # Download and check versions info @@ -76,7 +76,7 @@ def update_self(to_screen, verbose, opener): def version_tuple(version_str): return tuple(map(int, version_str.split('.'))) if version_tuple(__version__) >= version_tuple(version_id): - to_screen('youtube-dl is up to date (%s)' % __version__) + to_screen('youtube-dlc is up to date (%s)' % __version__) return to_screen('Updating to version ' + version_id + ' ...') @@ -126,14 +126,14 @@ def update_self(to_screen, verbose, opener): return try: - bat = os.path.join(directory, 'youtube-dl-updater.bat') + bat = os.path.join(directory, 'youtube-dlc-updater.bat') with io.open(bat, 'w') as batfile: batfile.write(''' @echo off echo Waiting for file handle to be closed ... ping 127.0.0.1 -n 5 -w 1000 > NUL move /Y "%s.new" "%s" > NUL -echo Updated youtube-dl to version %s. +echo Updated youtube-dlc to version %s. start /b "" cmd /c del "%%~f0"&exit /b" \n''' % (exe, exe, version_id)) @@ -171,7 +171,7 @@ start /b "" cmd /c del "%%~f0"&exit /b" to_screen('ERROR: unable to overwrite current version') return - to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.') + to_screen('Updated youtube-dlc. Restart youtube-dlc to use the new version.') def get_notes(versions, fromVersion): diff --git a/youtube_dl/utils.py b/youtube_dlc/utils.py index 01d9c0362..54a4ea2aa 100644 --- a/youtube_dl/utils.py +++ b/youtube_dlc/utils.py @@ -1984,6 +1984,7 @@ def get_elements_by_attribute(attribute, value, html, escape_value=True): class HTMLAttributeParser(compat_HTMLParser): """Trivial HTML parser to gather the attributes for a single element""" + def __init__(self): self.attrs = {} compat_HTMLParser.__init__(self) @@ -2317,12 +2318,12 @@ def make_HTTPS_handler(params, **kwargs): def bug_reports_message(): if ytdl_is_updateable(): - update_cmd = 'type youtube-dl -U to update' + update_cmd = 'type youtube-dlc -U to update' else: update_cmd = 'see https://yt-dl.org/update on how to update' msg = '; please report this issue on https://yt-dl.org/bug .' msg += ' Make sure you are using the latest version; %s.' % update_cmd - msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.' + msg += ' Be sure to call youtube-dlc with the --verbose flag and include its complete output.' return msg @@ -2336,7 +2337,7 @@ class ExtractorError(YoutubeDLError): def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None): """ tb, if given, is the original traceback (so that it can be printed out). - If expected is set, this is a normal error message and most likely not a bug in youtube-dl. + If expected is set, this is a normal error message and most likely not a bug in youtube-dlc. """ if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError): @@ -2378,6 +2379,7 @@ class GeoRestrictedError(ExtractorError): This exception may be thrown when a video is not available from your geographic location due to geographic restrictions imposed by a website. """ + def __init__(self, msg, countries=None): super(GeoRestrictedError, self).__init__(msg, expected=True) self.msg = msg @@ -2745,7 +2747,7 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar): _HTTPONLY_PREFIX = '#HttpOnly_' _ENTRY_LEN = 7 _HEADER = '''# Netscape HTTP Cookie File -# This file is generated by youtube-dl. Do not edit. +# This file is generated by youtube-dlc. Do not edit. ''' _CookieFileEntry = collections.namedtuple( @@ -3558,6 +3560,11 @@ def remove_quotes(s): return s +def get_domain(url): + domain = re.match(r'(?:https?:\/\/)?(?:www\.)?(?P<domain>[^\n\/]+\.[^\n\/]+)(?:\/(.*))?', url) + return domain.group('domain') if domain else None + + def url_basename(url): path = compat_urlparse.urlparse(url).path return path.strip('/').split('/')[-1] @@ -3732,7 +3739,7 @@ def get_exe_version(exe, args=['--version'], or False if the executable is not present """ try: # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers - # SIGTTOU if youtube-dl is run in the background. + # SIGTTOU if youtube-dlc is run in the background. # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656 out, _ = subprocess.Popen( [encodeArgument(exe)] + args, @@ -4144,7 +4151,7 @@ def is_outdated_version(version, limit, assume_new=True): def ytdl_is_updateable(): - """ Returns if youtube-dl can be updated with -U """ + """ Returns if youtube-dlc can be updated with -U """ from zipimport import zipimporter return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen') @@ -4173,6 +4180,7 @@ def mimetype2ext(mt): # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as # it's the most popular one 'audio/mpeg': 'mp3', + 'audio/x-wav': 'wav', }.get(mt) if ext is not None: return ext @@ -5353,7 +5361,7 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): return None # No Proxy if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'): req.add_header('Ytdl-socks-proxy', proxy) - # youtube-dl's http/https handlers do wrapping the socket with socks + # youtube-dlc's http/https handlers do wrapping the socket with socks return None return compat_urllib_request.ProxyHandler.proxy_open( self, req, proxy, type) @@ -5626,7 +5634,7 @@ def write_xattr(path, key, value): # TODO: fallback to CLI tools raise XAttrUnavailableError( 'python-pyxattr is detected but is too old. ' - 'youtube-dl requires %s or above while your version is %s. ' + 'youtube-dlc requires %s or above while your version is %s. ' 'Falling back to other xattr implementations' % ( pyxattr_required_version, xattr.__version__)) diff --git a/youtube_dl/version.py b/youtube_dlc/version.py index 45b4d3291..cbd52462c 100644 --- a/youtube_dl/version.py +++ b/youtube_dlc/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2020.09.06' +__version__ = '2020.09.29' |