[youtube] Extract comments' approximate timestamp (#221)

Authored by: colethedj

1 files changed, 26 insertions, 15 deletions

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 016750a70..c3d06b967 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -2,6 +2,7 @@
 
 from __future__ import unicode_literals
 
+import calendar
 import hashlib
 import itertools
 import json
@@ -27,6 +28,7 @@ from ..utils import (
     bool_or_none,
     clean_html,
     dict_get,
+    datetime_from_str,
     ExtractorError,
     format_field,
     float_or_none,
@@ -46,7 +48,7 @@ from ..utils import (
     update_url_query,
     url_or_none,
     urlencode_postdata,
-    urljoin,
+    urljoin
 )
 
 
@@ -1500,6 +1502,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
              regex), webpage, name, default='{}'), video_id, fatal=False)
 
     @staticmethod
+    def parse_time_text(time_text):
+        """
+        Parse the comment time text
+        time_text is in the format 'X units ago (edited)'
+        """
+        time_text_split = time_text.split(' ')
+        if len(time_text_split) >= 3:
+            return datetime_from_str('now-%s%s' % (time_text_split[0], time_text_split[1]), precision='auto')
+
+    @staticmethod
     def _join_text_entries(runs):
         text = None
         for run in runs:
@@ -1521,7 +1533,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         text = self._join_text_entries(comment_text_runs) or ''
         comment_time_text = try_get(comment_renderer, lambda x: x['publishedTimeText']['runs']) or []
         time_text = self._join_text_entries(comment_time_text)
-
+        timestamp = calendar.timegm(self.parse_time_text(time_text).timetuple())
         author = try_get(comment_renderer, lambda x: x['authorText']['simpleText'], compat_str)
         author_id = try_get(comment_renderer,
                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
@@ -1532,11 +1544,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
         is_liked = try_get(comment_renderer, lambda x: x['isLiked'], bool)
-
         return {
             'id': comment_id,
             'text': text,
-            # TODO: This should be parsed to timestamp
+            'timestamp': timestamp,
             'time_text': time_text,
             'like_count': votes,
             'is_favorited': is_liked,
@@ -1624,12 +1635,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     comment_prog_str = '(%d/%d)' % (comment_counts[0], comment_counts[1])
                     if page_num == 0:
                         if first_continuation:
-                            note_prefix = "Downloading initial comment continuation page"
+                            note_prefix = 'Downloading initial comment continuation page'
                         else:
-                            note_prefix = "    Downloading comment reply thread %d %s" % (comment_counts[2], comment_prog_str)
+                            note_prefix = '    Downloading comment reply thread %d %s' % (comment_counts[2], comment_prog_str)
                     else:
-                        note_prefix = "%sDownloading comment%s page %d %s" % (
-                            "       " if parent else "",
+                        note_prefix = '%sDownloading comment%s page %d %s' % (
+                            '       ' if parent else '',
                             ' replies' if parent else '',
                             page_num,
                             comment_prog_str)
@@ -1644,13 +1655,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 except ExtractorError as e:
                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503, 404, 413):
                         if e.cause.code == 413:
-                            self.report_warning("Assumed end of comments (received HTTP Error 413)")
+                            self.report_warning('Assumed end of comments (received HTTP Error 413)')
                             return
                         # Downloading page may result in intermittent 5xx HTTP error
                         # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
                         last_error = 'HTTP Error %s' % e.cause.code
                         if e.cause.code == 404:
-                            last_error = last_error + " (this API is probably deprecated)"
+                            last_error = last_error + ' (this API is probably deprecated)'
                         if count < retries:
                             continue
                     raise
@@ -1668,7 +1679,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
                     # YouTube sometimes gives reload: now json if something went wrong (e.g. bad auth)
                     if browse.get('reload'):
-                        raise ExtractorError("Invalid or missing params in continuation request", expected=False)
+                        raise ExtractorError('Invalid or missing params in continuation request', expected=False)
 
                     # TODO: not tested, merged from old extractor
                     err_msg = browse.get('externalErrorMessage')
@@ -1708,7 +1719,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
                     if expected_comment_count:
                         comment_counts[1] = str_to_int(expected_comment_count)
-                        self.to_screen("Downloading ~%d comments" % str_to_int(expected_comment_count))
+                        self.to_screen('Downloading ~%d comments' % str_to_int(expected_comment_count))
                         yield comment_counts[1]
 
                     # TODO: cli arg.
@@ -1724,7 +1735,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         continuation = YoutubeTabIE._build_continuation_query(
                             continuation=sort_continuation_renderer.get('continuation'),
                             ctp=sort_continuation_renderer.get('clickTrackingParams'))
-                        self.to_screen("Sorting comments by %s" % ('popular' if comment_sort_index == 0 else 'newest'))
+                        self.to_screen('Sorting comments by %s' % ('popular' if comment_sort_index == 0 else 'newest'))
                         break
 
                 for entry in known_continuation_renderers[key](continuation_renderer):
@@ -1757,7 +1768,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         continue
                     comments.append(comment)
                 break
-        self.to_screen("Downloaded %d/%d comments" % (len(comments), estimated_total))
+        self.to_screen('Downloaded %d/%d comments' % (len(comments), estimated_total))
         return {
             'comments': comments,
             'comment_count': len(comments),
@@ -2979,7 +2990,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                     self.report_warning('%s. Retrying ...' % last_error)
                 try:
                     response = self._call_api(
-                        ep="browse", fatal=True, headers=headers,
+                        ep='browse', fatal=True, headers=headers,
                         video_id='%s page %s' % (item_id, page_num),
                         query={
                             'continuation': continuation['continuation'],