diff options
Diffstat (limited to 'youtube/playlist.py')
| -rw-r--r-- | youtube/playlist.py | 243 |
1 files changed, 243 insertions, 0 deletions
diff --git a/youtube/playlist.py b/youtube/playlist.py new file mode 100644 index 0000000..fc09191 --- /dev/null +++ b/youtube/playlist.py @@ -0,0 +1,243 @@ +import base64
+import youtube.common as common
+import urllib
+import json
+from string import Template
+import youtube.proto as proto
+import gevent
+import math
+
+with open("yt_playlist_template.html", "r") as file:
+ yt_playlist_template = Template(file.read())
+
+
+
+
+
+
+def youtube_obfuscated_endian(offset):
+ if offset < 128:
+ return bytes((offset,))
+ first_byte = 255 & offset
+ second_byte = 255 & (offset >> 7)
+ second_byte = second_byte | 1
+
+ # The next 2 bytes encode the offset in little endian order,
+ # BUT, it's done in a strange way. The least significant bit (LSB) of the second byte is not part
+ # of the offset. Instead, to get the number which the two bytes encode, that LSB
+ # of the second byte is combined with the most significant bit (MSB) of the first byte
+ # in a logical AND. Replace the two bits with the result of the AND to get the two little endian
+ # bytes that represent the offset.
+
+ return bytes((first_byte, second_byte))
+
+
+
+# just some garbage that's required, don't know what it means, if it means anything.
+ctoken_header = b'\xe2\xa9\x85\xb2\x02' # e2 a9 85 b2 02
+
+def byte(x):
+ return bytes((x,))
+
+# TL;DR: the offset is hidden inside 3 nested base 64 encodes with random junk data added on the side periodically
+def create_ctoken(playlist_id, offset):
+ obfuscated_offset = b'\x08' + youtube_obfuscated_endian(offset) # 0x08 slapped on for no apparent reason
+ obfuscated_offset = b'PT:' + base64.urlsafe_b64encode(obfuscated_offset).replace(b'=', b'')
+ obfuscated_offset = b'z' + byte(len(obfuscated_offset)) + obfuscated_offset
+ obfuscated_offset = base64.urlsafe_b64encode(obfuscated_offset).replace(b'=', b'%3D')
+
+ playlist_bytes = b'VL' + bytes(playlist_id, 'ascii')
+ main_info = b'\x12' + byte(len(playlist_bytes)) + playlist_bytes + b'\x1a' + byte(len(obfuscated_offset)) + obfuscated_offset
+
+ ctoken = base64.urlsafe_b64encode(ctoken_header + byte(len(main_info)) + main_info)
+
+ return ctoken.decode('ascii')
+
+def playlist_ctoken(playlist_id, offset):
+
+ offset = proto.uint(1, offset)
+ # this is just obfuscation as far as I can tell. It doesn't even follow protobuf
+ offset = b'PT:' + proto.unpadded_b64encode(offset)
+ offset = proto.string(15, offset)
+
+ continuation_info = proto.string( 3, proto.percent_b64encode(offset) )
+
+ playlist_id = proto.string(2, 'VL' + playlist_id )
+ pointless_nest = proto.string(80226972, playlist_id + continuation_info)
+
+ return base64.urlsafe_b64encode(pointless_nest).decode('ascii')
+
+# initial request types:
+# polymer_json: https://m.youtube.com/playlist?list=PLv3TTBr1W_9tppikBxAE_G6qjWdBljBHJ&pbj=1&lact=0
+# ajax json: https://m.youtube.com/playlist?list=PLv3TTBr1W_9tppikBxAE_G6qjWdBljBHJ&pbj=1&lact=0 with header X-YouTube-Client-Version: 1.20180418
+
+
+# continuation request types:
+# polymer_json: https://m.youtube.com/playlist?&ctoken=[...]&pbj=1
+# ajax json: https://m.youtube.com/playlist?action_continuation=1&ajax=1&ctoken=[...]
+
+
+headers_1 = (
+ ('Accept', '*/*'),
+ ('Accept-Language', 'en-US,en;q=0.5'),
+ ('X-YouTube-Client-Name', '1'),
+ ('X-YouTube-Client-Version', '2.20180614'),
+)
+
+def playlist_first_page(playlist_id):
+ url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&ajax=1&disable_polymer=true'
+ content = common.fetch_url(url, common.mobile_ua + headers_1)
+ if content[0:4] == b")]}'":
+ content = content[4:]
+ content = json.loads(common.uppercase_escape(content.decode('utf-8')))
+ return content
+
+ajax_info_dispatch = {
+ 'view_count_text': ('views', common.get_text),
+ 'num_videos_text': ('size', lambda node: common.get_text(node).split(' ')[0]),
+ 'thumbnail': ('thumbnail', lambda node: node.url),
+ 'title': ('title', common.get_text),
+ 'owner_text': ('author', common.get_text),
+ 'owner_endpoint': ('author_url', lambda node: node.url),
+ 'description': ('description', common.get_formatted_text),
+
+}
+def metadata_info(ajax_json):
+ info = {}
+ try:
+ for key, node in ajax_json.items():
+ try:
+ simple_key, function = dispatch[key]
+ except KeyError:
+ continue
+ info[simple_key] = function(node)
+ return info
+ except (KeyError,IndexError):
+ print(ajax_json)
+ raise
+
+
+
+
+#https://m.youtube.com/playlist?itct=CBMQybcCIhMIptj9xJaJ2wIV2JKcCh3Idwu-&ctoken=4qmFsgI2EiRWTFBMT3kwajlBdmxWWlB0bzZJa2pLZnB1MFNjeC0tN1BHVEMaDmVnWlFWRHBEUWxFJTNE&pbj=1
+def get_videos_ajax(playlist_id, page):
+
+ url = "https://m.youtube.com/playlist?action_continuation=1&ajax=1&ctoken=" + playlist_ctoken(playlist_id, (int(page)-1)*20)
+ headers = {
+ 'User-Agent': ' Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1',
+ 'Accept': '*/*',
+ 'Accept-Language': 'en-US,en;q=0.5',
+ 'X-YouTube-Client-Name': '2',
+ 'X-YouTube-Client-Version': '1.20180508',
+ }
+ print("Sending playlist ajax request")
+ content = common.fetch_url(url, headers)
+ with open('playlist_debug', 'wb') as f:
+ f.write(content)
+ content = content[4:]
+ print("Finished recieving playlist response")
+
+ info = json.loads(common.uppercase_escape(content.decode('utf-8')))
+ return info
+
+def get_playlist_videos(ajax_json):
+ videos = []
+ #info = get_bloated_playlist_videos(playlist_id, page)
+ #print(info)
+ video_list = ajax_json['content']['continuation_contents']['contents']
+
+
+ for video_json_crap in video_list:
+ try:
+ videos.append({
+ "title": video_json_crap["title"]['runs'][0]['text'],
+ "id": video_json_crap["video_id"],
+ "views": "",
+ "duration": common.default_multi_get(video_json_crap, 'length', 'runs', 0, 'text', default=''), # livestreams dont have a length
+ "author": video_json_crap['short_byline']['runs'][0]['text'],
+ "author_url": '',
+ "published": '',
+ 'playlist_index': '',
+
+ })
+ except (KeyError, IndexError):
+ print(video_json_crap)
+ raise
+ return videos
+
+def get_playlist_videos_format2(playlist_id, page):
+ videos = []
+ info = get_bloated_playlist_videos(playlist_id, page)
+ video_list = info['response']['continuationContents']['playlistVideoListContinuation']['contents']
+
+ for video_json_crap in video_list:
+
+ video_json_crap = video_json_crap['videoRenderer']
+
+ try:
+ videos.append({
+ "title": video_json_crap["title"]['runs'][0]['text'],
+ "video_id": video_json_crap["videoId"],
+ "views": "",
+ "duration": common.default_multi_get(video_json_crap, 'lengthText', 'runs', 0, 'text', default=''), # livestreams dont have a length
+ "uploader": video_json_crap['shortBylineText']['runs'][0]['text'],
+ "uploader_url": common.ORIGIN_URL + video_json_crap['shortBylineText']['runs'][0]['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
+ "published": common.default_multi_get(video_json_crap, 'publishedTimeText', 'simpleText', default=''),
+ 'playlist_index': video_json_crap['index']['runs'][0]['text'],
+
+ })
+ except (KeyError, IndexError):
+ print(video_json_crap)
+ raise
+ return videos
+
+
+def playlist_videos_html(ajax_json):
+ result = ''
+ for info in get_playlist_videos(ajax_json):
+ result += common.small_video_item_html(info)
+ return result
+
+playlist_stat_template = Template('''
+<div>$stat</div>''')
+def get_playlist_page(query_string):
+ parameters = urllib.parse.parse_qs(query_string)
+ playlist_id = parameters['list'][0]
+ page = parameters.get("page", "1")[0]
+ if page == "1":
+ first_page_json = playlist_first_page(playlist_id)
+ this_page_json = first_page_json
+ else:
+ tasks = (
+ gevent.spawn(playlist_first_page, playlist_id ),
+ gevent.spawn(get_videos_ajax, playlist_id, page)
+ )
+ gevent.joinall(tasks)
+ first_page_json, this_page_json = tasks[0].value, tasks[1].value
+
+ try:
+ video_list = this_page_json['content']['section_list']['contents'][0]['contents'][0]['contents']
+ except KeyError:
+ video_list = this_page_json['content']['continuation_contents']['contents']
+ videos_html = ''
+ for video_json in video_list:
+ info = common.ajax_info(video_json)
+ videos_html += common.video_item_html(info, common.small_video_item_template)
+
+
+ metadata = common.ajax_info(first_page_json['content']['playlist_header'])
+ video_count = int(metadata['size'].replace(',', ''))
+ page_buttons = common.page_buttons_html(int(page), math.ceil(video_count/20), common.URL_ORIGIN + "/playlist", query_string)
+
+ html_ready = common.get_html_ready(metadata)
+ html_ready['page_title'] = html_ready['title'] + ' - Page ' + str(page)
+
+ stats = ''
+ stats += playlist_stat_template.substitute(stat=html_ready['size'] + ' videos')
+ stats += playlist_stat_template.substitute(stat=html_ready['views'])
+ return yt_playlist_template.substitute(
+ videos = videos_html,
+ page_buttons = page_buttons,
+ stats = stats,
+ **html_ready
+ )
\ No newline at end of file |