diff options
| -rw-r--r-- | youtube/comments.py | 29 | ||||
| -rw-r--r-- | youtube/proto.py | 68 |
2 files changed, 91 insertions, 6 deletions
diff --git a/youtube/comments.py b/youtube/comments.py index 3f44758..695d18a 100644 --- a/youtube/comments.py +++ b/youtube/comments.py @@ -59,6 +59,21 @@ def make_comment_ctoken(video_id, sort=0, offset=0, secret_key=''): result = proto.nested(2, proto.string(2, video_id)) + proto.uint(3,6) + proto.nested(6, offset_information) return base64.urlsafe_b64encode(result).decode('ascii') +def comment_replies_ctoken(video_id, comment_id, max_results=500): + + params = proto.string(2, comment_id) + proto.uint(9, max_results) + params = proto.nested(3, params) + + result = proto.nested(2, proto.string(2, video_id)) + proto.uint(3,6) + proto.nested(6, params) + return base64.urlsafe_b64encode(result).decode('ascii') + +def get_ids(ctoken): + params = proto.parse(proto.b64_to_bytes(ctoken)) + video_id = proto.parse(params[2])[2] + params = proto.parse(params[6]) + params = proto.parse(params[3]) + return params[2].decode('ascii'), video_id.decode('ascii') + mobile_headers = { 'Host': 'm.youtube.com', 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1', @@ -83,7 +98,7 @@ def request_comments(ctoken, replies=False): print("got <!DOCTYPE>, retrying") continue break - '''with open('comments_debug', 'wb') as f: + '''with open('debug/comments_debug', 'wb') as f: f.write(content)''' return content @@ -100,7 +115,8 @@ def parse_comments(content, replies=False): if not replies: if comment_raw['replies'] is not None: ctoken = comment_raw['replies']['continuations'][0]['continuation'] - replies_url = URL_ORIGIN + '/comments?ctoken=' + ctoken + "&replies=1" + comment_id, video_id = get_ids(ctoken) + replies_url = URL_ORIGIN + '/comments?parent_id=' + comment_id + "&video_id=" + video_id comment_raw = comment_raw['comment'] comment = { 'author': comment_raw['author']['runs'][0]['text'], @@ -148,8 +164,13 @@ more_comments_template = Template('''<a class="page-button more-comments" href=" def get_comments_page(query_string): parameters = urllib.parse.parse_qs(query_string) - ctoken = parameters['ctoken'][0] - replies = default_multi_get(parameters, 'replies', 0, default="0") == "1" + ctoken = default_multi_get(parameters, 'ctoken', 0, default='') + if not ctoken: + video_id = parameters['video_id'][0] + parent_id = parameters['parent_id'][0] + + ctoken = comment_replies_ctoken(video_id, parent_id) + replies = True result = parse_comments(request_comments(ctoken, replies), replies) comments_html, ctoken = get_comments_html(result) diff --git a/youtube/proto.py b/youtube/proto.py index 6230e51..004375a 100644 --- a/youtube/proto.py +++ b/youtube/proto.py @@ -1,5 +1,6 @@ from math import ceil import base64 +import io def byte(n): return bytes((n,)) @@ -61,5 +62,68 @@ def as_bytes(value): if isinstance(value, str): return value.encode('ascii') return value - -
\ No newline at end of file + + +def read_varint(data): + result = 0 + i = 0 + while True: + try: + byte = data.read(1)[0] + except IndexError: + if i == 0: + raise EOFError() + raise Exception('Unterminated varint starting at ' + str(data.tell() - i)) + result |= (byte & 127) << 7*i + if not byte & 128: + break + + i += 1 + return result + + +def read_group(data, end_sequence): + start = data.tell() + index = data.original.find(end_sequence, start) + if index == -1: + raise Exception('Unterminated group') + data.seek(index + len(end_sequence)) + return data.original[start:index] + +def read_protobuf(data): + data_original = data + data = io.BytesIO(data) + data.original = data_original + while True: + try: + tag = read_varint(data) + except EOFError: + break + wire_type = tag & 7 + field_number = tag >> 3 + + if wire_type == 0: + value = read_varint(data) + elif wire_type == 1: + value = data.read(8) + elif wire_type == 2: + length = read_varint(data) + value = data.read(length) + elif wire_type == 3: + end_bytes = encode_varint((field_number << 3) | 4) + value = read_group(data, end_bytes) + elif wire_type == 5: + value = data.read(4) + else: + raise Exception("Unknown wire type: " + str(wire_type) + ", Tag: " + bytes_to_hex(succinct_encode(tag)) + ", at position " + str(data.tell())) + yield (wire_type, field_number, value) + +def parse(data): + return {field_number: value for _, field_number, value in read_protobuf(data)} + +def b64_to_bytes(data): + if isinstance(data, bytes): + data = data.decode('ascii') + data = data.replace("%3D", "=") + return base64.urlsafe_b64decode(data + "="*((4 - len(data)%4)%4) ) + |