aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitea/workflows/ci.yaml23
-rw-r--r--.gitea/workflows/git-sync.yaml40
-rw-r--r--requirements-dev.txt6
-rw-r--r--requirements.txt6
-rw-r--r--settings.py18
-rw-r--r--youtube/channel.py2
-rw-r--r--youtube/playlist.py2
-rw-r--r--youtube/util.py115
-rw-r--r--youtube/version.py2
-rw-r--r--youtube/watch.py9
10 files changed, 171 insertions, 52 deletions
diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
new file mode 100644
index 0000000..d27f068
--- /dev/null
+++ b/.gitea/workflows/ci.yaml
@@ -0,0 +1,23 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: 3.11
+
+ - name: Install dependencies
+ run: |
+ pip install --upgrade pip
+ pip install -r requirements-dev.txt
+
+ - name: Run tests
+ run: pytest
diff --git a/.gitea/workflows/git-sync.yaml b/.gitea/workflows/git-sync.yaml
new file mode 100644
index 0000000..706df7e
--- /dev/null
+++ b/.gitea/workflows/git-sync.yaml
@@ -0,0 +1,40 @@
+name: git-sync-with-mirror
+
+on:
+ push:
+ branches: [ master ]
+ workflow_dispatch:
+
+jobs:
+ git-sync:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: git-sync
+ env:
+ git_sync_source_repo: git@git.fridu.us:heckyel/yt-local.git
+ git_sync_destination_repo: ssh://git@c.fridu.us/software/yt-local.git
+ if: env.git_sync_source_repo && env.git_sync_destination_repo
+ uses: astounds/git-sync@v1
+ with:
+ source_repo: git@git.fridu.us:heckyel/yt-local.git
+ source_branch: "master"
+ destination_repo: ssh://git@c.fridu.us/software/yt-local.git
+ destination_branch: "master"
+ source_ssh_private_key: ${{ secrets.GIT_SYNC_SOURCE_SSH_PRIVATE_KEY }}
+ destination_ssh_private_key: ${{ secrets.GIT_SYNC_DESTINATION_SSH_PRIVATE_KEY }}
+
+ - name: git-sync-sourcehut
+ env:
+ git_sync_source_repo: git@git.fridu.us:heckyel/yt-local.git
+ git_sync_destination_repo: git@git.sr.ht:~heckyel/yt-local
+ if: env.git_sync_source_repo && env.git_sync_destination_repo
+ uses: astounds/git-sync@v1
+ with:
+ source_repo: git@git.fridu.us:heckyel/yt-local.git
+ source_branch: "master"
+ destination_repo: git@git.sr.ht:~heckyel/yt-local
+ destination_branch: "master"
+ source_ssh_private_key: ${{ secrets.GIT_SYNC_SOURCE_SSH_PRIVATE_KEY }}
+ destination_ssh_private_key: ${{ secrets.GIT_SYNC_DESTINATION_SSH_PRIVATE_KEY }}
+ continue-on-error: true
diff --git a/requirements-dev.txt b/requirements-dev.txt
index fbeeed4..8208c56 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -8,14 +8,14 @@ gevent==24.2.1
greenlet==3.0.3
iniconfig==2.0.0
itsdangerous==2.1.2
-Jinja2==3.1.3
+Jinja2==3.1.4
MarkupSafe==2.1.5
packaging==24.0
pluggy==1.4.0
PySocks==1.7.1
pytest==8.1.1
stem==1.8.2
-urllib3==2.2.1
-Werkzeug==3.0.1
+urllib3==2.2.2
+Werkzeug==3.0.3
zope.event==5.0
zope.interface==6.2
diff --git a/requirements.txt b/requirements.txt
index 56867bb..95f8f12 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,11 +7,11 @@ Flask==3.0.2
gevent==24.2.1
greenlet==3.0.3
itsdangerous==2.1.2
-Jinja2==3.1.3
+Jinja2==3.1.4
MarkupSafe==2.1.5
PySocks==1.7.1
stem==1.8.2
-urllib3==2.2.1
-Werkzeug==3.0.1
+urllib3==2.2.2
+Werkzeug==3.0.3
zope.event==5.0
zope.interface==6.2
diff --git a/settings.py b/settings.py
index eb210c5..2de5efa 100644
--- a/settings.py
+++ b/settings.py
@@ -322,13 +322,6 @@ Archive: https://archive.ph/OZQbN''',
'comment': '',
}),
- ('gather_googlevideo_domains', {
- 'type': bool,
- 'default': False,
- 'comment': '''Developer use to debug 403s''',
- 'hidden': True,
- }),
-
('debugging_save_responses', {
'type': bool,
'default': False,
@@ -338,7 +331,7 @@ Archive: https://archive.ph/OZQbN''',
('settings_version', {
'type': int,
- 'default': 5,
+ 'default': 6,
'comment': '''Do not change, remove, or comment out this value, or else your settings may be lost or corrupted''',
'hidden': True,
}),
@@ -419,11 +412,20 @@ def upgrade_to_5(settings_dict):
return new_settings
+def upgrade_to_6(settings_dict):
+ new_settings = settings_dict.copy()
+ if 'gather_googlevideo_domains' in new_settings:
+ del new_settings['gather_googlevideo_domains']
+ new_settings['settings_version'] = 6
+ return new_settings
+
+
upgrade_functions = {
1: upgrade_to_2,
2: upgrade_to_3,
3: upgrade_to_4,
4: upgrade_to_5,
+ 5: upgrade_to_6,
}
diff --git a/youtube/channel.py b/youtube/channel.py
index b520121..81881eb 100644
--- a/youtube/channel.py
+++ b/youtube/channel.py
@@ -292,7 +292,7 @@ def get_number_of_videos_channel(channel_id):
try:
response = util.fetch_url(url, headers_mobile,
debug_name='number_of_videos', report_text='Got number of videos')
- except urllib.error.HTTPError as e:
+ except (urllib.error.HTTPError, util.FetchError) as e:
traceback.print_exc()
print("Couldn't retrieve number of videos")
return 1000
diff --git a/youtube/playlist.py b/youtube/playlist.py
index 83d530c..28b8149 100644
--- a/youtube/playlist.py
+++ b/youtube/playlist.py
@@ -115,7 +115,7 @@ def get_playlist_page():
video_count = yt_data_extract.deep_get(info, 'metadata', 'video_count')
if video_count is None:
- video_count = 40
+ video_count = 1000
return flask.render_template(
'playlist.html',
diff --git a/youtube/util.py b/youtube/util.py
index 0416fc1..eb121e1 100644
--- a/youtube/util.py
+++ b/youtube/util.py
@@ -321,7 +321,8 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
response.getheader('Content-Encoding', default='identity'))
if (settings.debugging_save_responses
- and debug_name is not None and content):
+ and debug_name is not None
+ and content):
save_dir = os.path.join(settings.data_dir, 'debug')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
@@ -394,23 +395,22 @@ def head(url, use_tor=False, report_text=None, max_redirects=10):
round(time.monotonic() - start_time, 3))
return response
-
-mobile_user_agent = 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.80 Mobile Safari/537.36'
+mobile_user_agent = 'Mozilla/5.0 (Linux; Android 7.0; Redmi Note 4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36'
mobile_ua = (('User-Agent', mobile_user_agent),)
-desktop_user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0'
+desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0'
desktop_ua = (('User-Agent', desktop_user_agent),)
json_header = (('Content-Type', 'application/json'),)
desktop_xhr_headers = (
('Accept', '*/*'),
('Accept-Language', 'en-US,en;q=0.5'),
('X-YouTube-Client-Name', '1'),
- ('X-YouTube-Client-Version', '2.20240327.00.00'),
+ ('X-YouTube-Client-Version', '2.20240304.00.00'),
) + desktop_ua
mobile_xhr_headers = (
('Accept', '*/*'),
('Accept-Language', 'en-US,en;q=0.5'),
- ('X-YouTube-Client-Name', '1'),
- ('X-YouTube-Client-Version', '2.20240328.08.00'),
+ ('X-YouTube-Client-Name', '2'),
+ ('X-YouTube-Client-Version', '2.20240304.08.00'),
) + mobile_ua
@@ -667,39 +667,47 @@ def to_valid_filename(name):
# https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72
INNERTUBE_CLIENTS = {
- 'android_music': {
- 'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
+ 'android': {
+ 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
'INNERTUBE_CONTEXT': {
'client': {
'hl': 'en',
'gl': 'US',
- 'clientName': 'ANDROID_MUSIC',
- 'clientVersion': '6.44.54',
+ 'clientName': 'ANDROID',
+ 'clientVersion': '19.09.36',
'osName': 'Android',
- 'osVersion': '14',
- 'androidSdkVersion': 34,
+ 'osVersion': '12',
+ 'androidSdkVersion': 31,
'platform': 'MOBILE',
- 'userAgent': 'com.google.android.apps.youtube.music/6.44.54 (Linux; U; Android 14; US) gzip'
- }
+ 'userAgent': 'com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip'
+ },
+ # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
+ #'thirdParty': {
+ # 'embedUrl': 'https://google.com', # Can be any valid URL
+ #}
},
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
- 'REQUIRE_JS_PLAYER': False
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
+ 'REQUIRE_JS_PLAYER': False,
},
- 'android': {
+ 'android-test-suite': {
'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
'INNERTUBE_CONTEXT': {
'client': {
'hl': 'en',
'gl': 'US',
- 'clientName': 'ANDROID',
- 'clientVersion': '19.12.36',
+ 'clientName': 'ANDROID_TESTSUITE',
+ 'clientVersion': '1.9',
'osName': 'Android',
- 'osVersion': '14',
- 'androidSdkVersion': 34,
+ 'osVersion': '12',
+ 'androidSdkVersion': 31,
'platform': 'MOBILE',
- 'userAgent': 'com.google.android.youtube/19.13.36 (Linux; U; Android 14; en_US; Google Pixel 6 Pro) gzip'
- }
+ 'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip'
+ },
+ # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
+ #'thirdParty': {
+ # 'embedUrl': 'https://google.com', # Can be any valid URL
+ #}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
'REQUIRE_JS_PLAYER': False,
@@ -712,9 +720,9 @@ INNERTUBE_CLIENTS = {
'hl': 'en',
'gl': 'US',
'clientName': 'IOS',
- 'clientVersion': '19.12.3',
+ 'clientVersion': '19.09.3',
'deviceModel': 'iPhone14,3',
- 'userAgent': 'com.google.ios.youtube/19.12.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
+ 'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
@@ -748,14 +756,62 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB',
- 'clientVersion': '2.20240327.00.00',
+ 'clientVersion': '2.20220801.00.00',
'userAgent': desktop_user_agent,
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 1
},
+ 'android_vr': {
+ 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'ANDROID_VR',
+ 'clientVersion': '1.60.19',
+ 'deviceMake': 'Oculus',
+ 'deviceModel': 'Quest 3',
+ 'androidSdkVersion': 32,
+ 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
+ 'osName': 'Android',
+ 'osVersion': '12L',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
+ 'REQUIRE_JS_PLAYER': False,
+ },
}
+def get_visitor_data():
+ visitor_data = None
+ visitor_data_cache = os.path.join(settings.data_dir, 'visitorData.txt')
+ if not os.path.exists(settings.data_dir):
+ os.makedirs(settings.data_dir)
+ if os.path.isfile(visitor_data_cache):
+ with open(visitor_data_cache, 'r') as file:
+ print('Getting visitor_data from cache')
+ visitor_data = file.read()
+ max_age = 12*3600
+ file_age = time.time() - os.path.getmtime(visitor_data_cache)
+ if file_age > max_age:
+ print('visitor_data cache is too old. Removing file...')
+ os.remove(visitor_data_cache)
+ return visitor_data
+
+ print('Fetching youtube homepage to get visitor_data')
+ yt_homepage = 'https://www.youtube.com'
+ yt_resp = fetch_url(yt_homepage, headers={'User-Agent': mobile_user_agent}, report_text='Getting youtube homepage')
+ visitor_data_re = r'''"visitorData":\s*?"(.+?)"'''
+ visitor_data_match = re.search(visitor_data_re, yt_resp.decode())
+ if visitor_data_match:
+ visitor_data = visitor_data_match.group(1)
+ print(f'Got visitor_data: {len(visitor_data)}')
+ with open(visitor_data_cache, 'w') as file:
+ print('Saving visitor_data cache...')
+ file.write(visitor_data)
+ return visitor_data
+ else:
+ print('Unable to get visitor_data value')
+ return visitor_data
def call_youtube_api(client, api, data):
client_params = INNERTUBE_CLIENTS[client]
@@ -763,12 +819,17 @@ def call_youtube_api(client, api, data):
key = client_params['INNERTUBE_API_KEY']
host = client_params.get('INNERTUBE_HOST') or 'www.youtube.com'
user_agent = context['client'].get('userAgent') or mobile_user_agent
+ visitor_data = get_visitor_data()
url = 'https://' + host + '/youtubei/v1/' + api + '?key=' + key
+ if visitor_data:
+ context['client'].update({'visitorData': visitor_data})
data['context'] = context
data = json.dumps(data)
headers = (('Content-Type', 'application/json'),('User-Agent', user_agent))
+ if visitor_data:
+ headers = ( *headers, ('X-Goog-Visitor-Id', visitor_data ))
response = fetch_url(
url, data=data, headers=headers,
debug_name='youtubei_' + api + '_' + client,
diff --git a/youtube/version.py b/youtube/version.py
index 3679399..eaae66d 100644
--- a/youtube/version.py
+++ b/youtube/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '0.2.14'
+__version__ = '0.2.20'
diff --git a/youtube/watch.py b/youtube/watch.py
index 09a3b76..e2762cb 100644
--- a/youtube/watch.py
+++ b/youtube/watch.py
@@ -343,7 +343,6 @@ def _add_to_error(info, key, additional_message):
def fetch_player_response(client, video_id):
return util.call_youtube_api(client, 'player', {
'videoId': video_id,
- 'params': 'CgIIAQ==',
})
@@ -372,7 +371,7 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None):
tasks = (
# Get video metadata from here
gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
- gevent.spawn(fetch_player_response, 'android', video_id)
+ gevent.spawn(fetch_player_response, 'android_vr', video_id)
)
gevent.joinall(tasks)
util.check_gevent_exceptions(*tasks)
@@ -650,12 +649,6 @@ def get_watch_page(video_id=None):
'/videoplayback',
'/videoplayback/name/' + filename)
- if settings.gather_googlevideo_domains:
- with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f:
- url = info['formats'][0]['url']
- subdomain = url[0:url.find(".googlevideo.com")]
- f.write(subdomain + "\n")
-
download_formats = []
for format in (info['formats'] + info['hls_formats']):
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-28 07:47:59 +0000