diff options
| -rw-r--r-- | .github/workflows/python-publish.yml | 33 | ||||
| -rw-r--r-- | .travis.yml | 12 | ||||
| -rw-r--r-- | .travis.yml.original | 50 | ||||
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | setup.py | 127 | ||||
| -rw-r--r-- | setup_original.py | 148 | ||||
| -rw-r--r-- | youtube_dl/extractor/biqle.py | 20 | ||||
| -rw-r--r-- | youtube_dl/extractor/xhamster.py | 17 | ||||
| -rw-r--r-- | youtube_dl/version.py | 2 |
9 files changed, 301 insertions, 110 deletions
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 000000000..0fa2d1857 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,33 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + pull_request: + branches: + - release + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + rm -rf dist/* + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.travis.yml b/.travis.yml index 51afd469a..fb499845e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,29 +12,18 @@ python: dist: trusty env: - YTDL_TEST_SET=core - - YTDL_TEST_SET=download jobs: include: - python: 3.7 dist: xenial env: YTDL_TEST_SET=core - - python: 3.7 - dist: xenial - env: YTDL_TEST_SET=download - python: 3.8 dist: xenial env: YTDL_TEST_SET=core - - python: 3.8 - dist: xenial - env: YTDL_TEST_SET=download - python: 3.8-dev dist: xenial env: YTDL_TEST_SET=core - - python: 3.8-dev - dist: xenial - env: YTDL_TEST_SET=download - env: JYTHON=true; YTDL_TEST_SET=core - - env: JYTHON=true; YTDL_TEST_SET=download - name: flake8 python: 3.8 dist: xenial @@ -44,7 +33,6 @@ jobs: allow_failures: - env: YTDL_TEST_SET=download - env: JYTHON=true; YTDL_TEST_SET=core - - env: JYTHON=true; YTDL_TEST_SET=download before_install: - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi script: ./devscripts/run_tests.sh diff --git a/.travis.yml.original b/.travis.yml.original new file mode 100644 index 000000000..51afd469a --- /dev/null +++ b/.travis.yml.original @@ -0,0 +1,50 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.2" + - "3.3" + - "3.4" + - "3.5" + - "3.6" + - "pypy" + - "pypy3" +dist: trusty +env: + - YTDL_TEST_SET=core + - YTDL_TEST_SET=download +jobs: + include: + - python: 3.7 + dist: xenial + env: YTDL_TEST_SET=core + - python: 3.7 + dist: xenial + env: YTDL_TEST_SET=download + - python: 3.8 + dist: xenial + env: YTDL_TEST_SET=core + - python: 3.8 + dist: xenial + env: YTDL_TEST_SET=download + - python: 3.8-dev + dist: xenial + env: YTDL_TEST_SET=core + - python: 3.8-dev + dist: xenial + env: YTDL_TEST_SET=download + - env: JYTHON=true; YTDL_TEST_SET=core + - env: JYTHON=true; YTDL_TEST_SET=download + - name: flake8 + python: 3.8 + dist: xenial + install: pip install flake8 + script: flake8 . + fast_finish: true + allow_failures: + - env: YTDL_TEST_SET=download + - env: JYTHON=true; YTDL_TEST_SET=core + - env: JYTHON=true; YTDL_TEST_SET=download +before_install: + - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi +script: ./devscripts/run_tests.sh @@ -1,4 +1,4 @@ -[](https://travis-ci.org/ytdl-org/youtube-dl) +[](https://travis-ci.com/blackjack4494/youtube-dlc) youtube-dl - download videos from youtube.com or other video platforms @@ -1,62 +1,21 @@ #!/usr/bin/env python # coding: utf-8 -from __future__ import print_function - +from setuptools import setup, Command import os.path import warnings import sys - -try: - from setuptools import setup, Command - setuptools_available = True -except ImportError: - from distutils.core import setup, Command - setuptools_available = False from distutils.spawn import spawn -try: - # This will create an exe that needs Microsoft Visual C++ 2008 - # Redistributable Package - import py2exe -except ImportError: - if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': - print('Cannot import py2exe', file=sys.stderr) - exit(1) - -py2exe_options = { - 'bundle_files': 1, - 'compressed': 1, - 'optimize': 2, - 'dist_dir': '.', - 'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'], -} - # Get the version from youtube_dl/version.py without importing the package exec(compile(open('youtube_dl/version.py').read(), 'youtube_dl/version.py', 'exec')) -DESCRIPTION = 'YouTube video downloader' -LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites' - -py2exe_console = [{ - 'script': './youtube_dl/__main__.py', - 'dest_base': 'youtube-dl', - 'version': __version__, - 'description': DESCRIPTION, - 'comments': LONG_DESCRIPTION, - 'product_name': 'youtube-dl', - 'product_version': __version__, -}] - -py2exe_params = { - 'console': py2exe_console, - 'options': {'py2exe': py2exe_options}, - 'zipfile': None -} +DESCRIPTION = 'Media downloader supporting various sites such as youtube' +LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites. Based on a more active community fork.' if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': - params = py2exe_params + print("inv") else: files_spec = [ ('etc/bash_completion.d', ['youtube-dl.bash-completion']), @@ -78,10 +37,10 @@ else: params = { 'data_files': data_files, } - if setuptools_available: - params['entry_points'] = {'console_scripts': ['youtube-dl = youtube_dl:main']} - else: - params['scripts'] = ['bin/youtube-dl'] + #if setuptools_available: + params['entry_points'] = {'console_scripts': ['youtube-dlc = youtube_dl:main']} + #else: + # params['scripts'] = ['bin/youtube-dlc'] class build_lazy_extractors(Command): description = 'Build the extractor lazy loading module' @@ -100,49 +59,45 @@ class build_lazy_extractors(Command): ) setup( - name='youtube_dl', + name="youtube_dlc", version=__version__, + maintainer="Tom-Oliver Heidel", + maintainer_email="theidel@uni-bremen.de", description=DESCRIPTION, long_description=LONG_DESCRIPTION, - url='https://github.com/ytdl-org/youtube-dl', - author='Ricardo Garcia', - author_email='ytdl@yt-dl.org', - maintainer='Sergey M.', - maintainer_email='dstftw@gmail.com', - license='Unlicense', - packages=[ + # long_description_content_type="text/markdown", + url="https://github.com/blackjack4494/youtube-dlc", + # packages=setuptools.find_packages(), + packages=[ 'youtube_dl', 'youtube_dl.extractor', 'youtube_dl.downloader', 'youtube_dl.postprocessor'], - - # Provokes warning on most systems (why?!) - # test_suite = 'nose.collector', - # test_requires = ['nosetest'], - classifiers=[ - 'Topic :: Multimedia :: Video', - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'License :: Public Domain', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: Implementation', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: IronPython', - 'Programming Language :: Python :: Implementation :: Jython', - 'Programming Language :: Python :: Implementation :: PyPy', + "Topic :: Multimedia :: Video", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: IronPython", + "Programming Language :: Python :: Implementation :: Jython", + "Programming Language :: Python :: Implementation :: PyPy", + "License :: Public Domain", + "Operating System :: OS Independent", ], - - cmdclass={'build_lazy_extractors': build_lazy_extractors}, + python_requires='>=2.6', + + cmdclass={'build_lazy_extractors': build_lazy_extractors}, **params -) +)
\ No newline at end of file diff --git a/setup_original.py b/setup_original.py new file mode 100644 index 000000000..4138e7924 --- /dev/null +++ b/setup_original.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import print_function + +import os.path +import warnings +import sys + +try: + from setuptools import setup, Command + setuptools_available = True +except ImportError: + from distutils.core import setup, Command + setuptools_available = False +from distutils.spawn import spawn + +try: + # This will create an exe that needs Microsoft Visual C++ 2008 + # Redistributable Package + import py2exe +except ImportError: + if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': + print('Cannot import py2exe', file=sys.stderr) + exit(1) + +py2exe_options = { + 'bundle_files': 1, + 'compressed': 1, + 'optimize': 2, + 'dist_dir': '.', + 'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'], +} + +# Get the version from youtube_dl/version.py without importing the package +exec(compile(open('youtube_dl/version.py').read(), + 'youtube_dl/version.py', 'exec')) + +DESCRIPTION = 'Media downloader supporting various sites such as youtube' +LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites. Based on a more active community fork.' + +py2exe_console = [{ + 'script': './youtube_dl/__main__.py', + 'dest_base': 'youtube-dl', + 'version': __version__, + 'description': DESCRIPTION, + 'comments': LONG_DESCRIPTION, + 'product_name': 'youtube-dlc', + 'product_version': __version__, +}] + +py2exe_params = { + 'console': py2exe_console, + 'options': {'py2exe': py2exe_options}, + 'zipfile': None +} + +if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': + params = py2exe_params +else: + files_spec = [ + ('etc/bash_completion.d', ['youtube-dl.bash-completion']), + ('etc/fish/completions', ['youtube-dl.fish']), + ('share/doc/youtube_dl', ['README.txt']), + ('share/man/man1', ['youtube-dl.1']) + ] + root = os.path.dirname(os.path.abspath(__file__)) + data_files = [] + for dirname, files in files_spec: + resfiles = [] + for fn in files: + if not os.path.exists(fn): + warnings.warn('Skipping file %s since it is not present. Type make to build all automatically generated files.' % fn) + else: + resfiles.append(fn) + data_files.append((dirname, resfiles)) + + params = { + 'data_files': data_files, + } + if setuptools_available: + params['entry_points'] = {'console_scripts': ['youtube-dl = youtube_dl:main']} + else: + params['scripts'] = ['bin/youtube-dl'] + +class build_lazy_extractors(Command): + description = 'Build the extractor lazy loading module' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + spawn( + [sys.executable, 'devscripts/make_lazy_extractors.py', 'youtube_dl/extractor/lazy_extractors.py'], + dry_run=self.dry_run, + ) + +setup( + name='youtube_dlc', + version=__version__, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + url='https://github.com/blackjack4494/youtube-dlc', + author='', + author_email='theidel@uni-bremen.de', + maintainer='Tom-Oliver Heidel', + maintainer_email='theidel@uni-bremen.de', + license='Unlicense', + packages=[ + 'youtube_dl', + 'youtube_dl.extractor', 'youtube_dl.downloader', + 'youtube_dl.postprocessor'], + + # Provokes warning on most systems (why?!) + # test_suite = 'nose.collector', + # test_requires = ['nosetest'], + + classifiers=[ + 'Topic :: Multimedia :: Video', + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'License :: Public Domain', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: Implementation', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: IronPython', + 'Programming Language :: Python :: Implementation :: Jython', + 'Programming Language :: Python :: Implementation :: PyPy', + ], + + cmdclass={'build_lazy_extractors': build_lazy_extractors}, + **params +) diff --git a/youtube_dl/extractor/biqle.py b/youtube_dl/extractor/biqle.py index af21e3ee5..17ebbb257 100644 --- a/youtube_dl/extractor/biqle.py +++ b/youtube_dl/extractor/biqle.py @@ -3,10 +3,11 @@ from __future__ import unicode_literals from .common import InfoExtractor from .vk import VKIE -from ..utils import ( - HEADRequest, - int_or_none, +from ..compat import ( + compat_b64decode, + compat_urllib_parse_unquote, ) +from ..utils import int_or_none class BIQLEIE(InfoExtractor): @@ -47,9 +48,16 @@ class BIQLEIE(InfoExtractor): if VKIE.suitable(embed_url): return self.url_result(embed_url, VKIE.ie_key(), video_id) - self._request_webpage( - HEADRequest(embed_url), video_id, headers={'Referer': url}) - video_id, sig, _, access_token = self._get_cookies(embed_url)['video_ext'].value.split('%3A') + embed_page = self._download_webpage( + embed_url, video_id, headers={'Referer': url}) + video_ext = self._get_cookies(embed_url).get('video_ext') + if video_ext: + video_ext = compat_urllib_parse_unquote(video_ext.value) + if not video_ext: + video_ext = compat_b64decode(self._search_regex( + r'video_ext\s*:\s*[\'"]([A-Za-z0-9+/=]+)', + embed_page, 'video_ext')).decode() + video_id, sig, _, access_token = video_ext.split(':') item = self._download_json( 'https://api.vk.com/method/video.get', video_id, headers={'User-Agent': 'okhttp/3.4.1'}, query={ diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 0f7be6a7d..902a3ed33 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -20,13 +20,13 @@ from ..utils import ( class XHamsterIE(InfoExtractor): - _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster[27]\.com)' + _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com)' _VALID_URL = r'''(?x) https?:// (?:.+?\.)?%s/ (?: - movies/(?P<id>\d+)/(?P<display_id>[^/]*)\.html| - videos/(?P<display_id_2>[^/]*)-(?P<id_2>\d+) + movies/(?P<id>[\dA-Za-z]+)/(?P<display_id>[^/]*)\.html| + videos/(?P<display_id_2>[^/]*)-(?P<id_2>[\dA-Za-z]+) ) ''' % _DOMAINS _TESTS = [{ @@ -100,11 +100,20 @@ class XHamsterIE(InfoExtractor): 'url': 'https://xhamster2.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445', 'only_matching': True, }, { + 'url': 'https://xhamster11.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445', + 'only_matching': True, + }, { + 'url': 'https://xhamster26.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445', + 'only_matching': True, + }, { 'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html', 'only_matching': True, }, { 'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd', 'only_matching': True, + }, { + 'url': 'http://de.xhamster.com/videos/skinny-girl-fucks-herself-hard-in-the-forest-xhnBJZx', + 'only_matching': True, }] def _real_extract(self, url): @@ -129,7 +138,7 @@ class XHamsterIE(InfoExtractor): initials = self._parse_json( self._search_regex( - r'window\.initials\s*=\s*({.+?})\s*;\s*\n', webpage, 'initials', + r'window\.initials\s*=\s*({.+?})\s*;', webpage, 'initials', default='{}'), video_id, fatal=False) if initials: diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 17101fa47..b50bd2b3b 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2020.07.28' +__version__ = '2020.08.31' |