From aa3e5aa441b79a471f96080501c114fd9ad34ba5 Mon Sep 17 00:00:00 2001
From: James Taylor <user234683@users.noreply.github.com>
Date: Sun, 28 Jun 2020 17:52:24 -0700
Subject: Add dialog for copying urls to external player for livestreams Also
 for livestreams which are over whose other sources aren't present or aren't
 ready yet.

---
 youtube/templates/watch.html                | 30 ++++++++++++++
 youtube/watch.py                            | 19 ++++++++-
 youtube/yt_data_extract/__init__.py         |  2 +-
 youtube/yt_data_extract/watch_extraction.py | 62 ++++++++++++++++++++++++-----
 4 files changed, 101 insertions(+), 12 deletions(-)

(limited to 'youtube')

diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html
index c722115..04f963d 100644
--- a/youtube/templates/watch.html
+++ b/youtube/templates/watch.html
@@ -27,6 +27,27 @@
             transform: translate(-50%, -50%);
         }
 
+    .live-url-choices{
+        height: 360px;
+        width: 640px;
+        grid-column: 2;
+        background-color: var(--video-background-color);
+        padding: 25px 0px 0px 25px;
+    }
+        .live-url-choices ol{
+            list-style: none;
+            padding:0px;
+            margin:0px;
+            margin-top: 15px;
+        }
+        .live-url-choices input{
+            width: 400px;
+        }
+        .url-choice-label{
+            display: inline-block;
+            width: 150px;
+        }
+
     {% if theater_mode %}
         video{
             grid-column: 1 / span 5;
@@ -296,6 +317,15 @@ Reload without invidious (for usage of new identity button).</a>
             {% endif %}
             </span>
         </div>
+    {% elif (video_sources.__len__() == 0 or live) and hls_formats.__len__() != 0 %}
+        <div class="live-url-choices">
+            <span>Copy a url into your video player:</span>
+            <ol>
+                {% for fmt in hls_formats %}
+                    <li class="url-choice"><div class="url-choice-label">{{ fmt['video_quality'] }}: </div><input class="url-choice-copy" value="{{ fmt['url'] }}" readonly onclick="this.select();"></li>
+                {% endfor %}
+            </ol>
+        </div>
     {% else %}
         <video controls autofocus class="video">
             {% for video_source in video_sources %}
diff --git a/youtube/watch.py b/youtube/watch.py
index 929a1ac..b5636b8 100644
--- a/youtube/watch.py
+++ b/youtube/watch.py
@@ -242,6 +242,22 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None):
         decryption_error = 'Error decrypting url signatures: ' + decryption_error
         info['playability_error'] = decryption_error
 
+    # livestream urls
+    # sometimes only the livestream urls work soon after the livestream is over
+    if info['hls_manifest_url'] and (info['live'] or not info['formats']):
+        manifest = util.fetch_url(info['hls_manifest_url'],
+            debug_name='hls_manifest.m3u8',
+            report_text='Fetched hls manifest'
+        ).decode('utf-8')
+
+        info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
+        if not err:
+            info['playability_error'] = None
+        for fmt in info['hls_formats']:
+            fmt['video_quality'] = video_quality_string(fmt)
+    else:
+        info['hls_formats'] = []
+
     # check for 403
     info['invidious_used'] = False
     info['invidious_reload_button'] = False
@@ -396,7 +412,7 @@ def get_watch_page(video_id=None):
 
     download_formats = []
 
-    for format in info['formats']:
+    for format in (info['formats'] + info['hls_formats']):
         if format['acodec'] and format['vcodec']:
             codecs_string = format['acodec'] + ', ' + format['vcodec']
         else:
@@ -435,6 +451,7 @@ def get_watch_page(video_id=None):
         download_formats        = download_formats,
         video_info              = json.dumps(video_info),
         video_sources           = video_sources,
+        hls_formats             = info['hls_formats'],
         subtitle_sources        = get_subtitle_sources(info),
         related                 = info['related_videos'],
         playlist                = info['playlist'],
diff --git a/youtube/yt_data_extract/__init__.py b/youtube/yt_data_extract/__init__.py
index 3378b8d..8934f74 100644
--- a/youtube/yt_data_extract/__init__.py
+++ b/youtube/yt_data_extract/__init__.py
@@ -9,4 +9,4 @@ from .everything_else import (extract_channel_info, extract_search_info,
 from .watch_extraction import (extract_watch_info, get_caption_url,
     update_with_age_restricted_info, requires_decryption,
     extract_decryption_function, decrypt_signatures, _formats,
-    update_format_with_type_info)
+    update_format_with_type_info, extract_hls_formats)
diff --git a/youtube/yt_data_extract/watch_extraction.py b/youtube/yt_data_extract/watch_extraction.py
index 9dbb252..5aaa318 100644
--- a/youtube/yt_data_extract/watch_extraction.py
+++ b/youtube/yt_data_extract/watch_extraction.py
@@ -307,6 +307,18 @@ def _extract_watch_info_desktop(top_level):
 
     return info
 
+def update_format_with_codec_info(fmt, codec):
+    if (codec.startswith('av')
+            or codec in ('vp9', 'vp8', 'vp8.0', 'h263', 'h264', 'mp4v')):
+        if codec == 'vp8.0':
+            codec = 'vp8'
+        conservative_update(fmt, 'vcodec', codec)
+    elif (codec.startswith('mp4a')
+            or codec in ('opus', 'mp3', 'aac', 'dtse', 'ec-3', 'vorbis')):
+        conservative_update(fmt, 'acodec', codec)
+    else:
+        print('Warning: unrecognized codec: ' + codec)
+
 fmt_type_re = re.compile(
     r'(text|audio|video)/([\w0-9]+); codecs="([\w0-9\.]+(?:, [\w0-9\.]+)*)"')
 def update_format_with_type_info(fmt, yt_fmt):
@@ -319,16 +331,7 @@ def update_format_with_type_info(fmt, yt_fmt):
     type, fmt['ext'], codecs = match.groups()
     codecs = codecs.split(', ')
     for codec in codecs:
-        if (codec.startswith('av')
-                or codec in ('vp9', 'vp8', 'vp8.0', 'h263', 'h264', 'mp4v')):
-            if codec == 'vp8.0':
-                codec = 'vp8'
-            conservative_update(fmt, 'vcodec', codec)
-        elif (codec.startswith('mp4a')
-                or codec in ('opus', 'mp3', 'aac', 'dtse', 'ec-3', 'vorbis')):
-            conservative_update(fmt, 'acodec', codec)
-        else:
-            print('Warning: unrecognized codec: ' + codec)
+        update_format_with_codec_info(fmt, codec)
     if type == 'audio':
         assert len(codecs) == 1
 
@@ -337,6 +340,8 @@ def _extract_formats(info, player_response):
     yt_formats = streaming_data.get('formats', []) + streaming_data.get('adaptiveFormats', [])
 
     info['formats'] = []
+    info['hls_manifest_url'] = streaming_data.get('hlsManifestUrl')
+    info['dash_manifest_url'] = streaming_data.get('dashManifestUrl')
 
     for yt_fmt in yt_formats:
         fmt = {}
@@ -371,6 +376,43 @@ def _extract_formats(info, player_response):
     else:
         info['ip_address'] = None
 
+hls_regex = re.compile(r'[\w_-]+=(?:"[^"]+"|[^",]+),')
+def extract_hls_formats(hls_manifest):
+    '''returns hls_formats, err'''
+    hls_formats = []
+    try:
+        lines = hls_manifest.splitlines()
+        i = 0
+        while i < len(lines):
+            if lines[i].startswith('#EXT-X-STREAM-INF'):
+                fmt = {'acodec': None, 'vcodec': None, 'height': None,
+                    'width': None, 'fps': None, 'audio_bitrate': None,
+                    'itag': None, 'file_size': None,
+                    'audio_sample_rate': None, 'url': None}
+                properties = lines[i].split(':')[1]
+                properties += ',' # make regex work for last key-value pair
+
+                for pair in hls_regex.findall(properties):
+                    key, value = pair.rstrip(',').split('=')
+                    if key == 'CODECS':
+                        for codec in value.strip('"').split(','):
+                            update_format_with_codec_info(fmt, codec)
+                    elif key == 'RESOLUTION':
+                        fmt['width'], fmt['height'] = map(int, value.split('x'))
+                        fmt['resolution'] = value
+                    elif key == 'FRAME-RATE':
+                        fmt['fps'] = int(value)
+                i += 1
+                fmt['url'] = lines[i]
+                assert fmt['url'].startswith('http')
+                fmt['ext'] = 'm3u8'
+                hls_formats.append(fmt)
+            i += 1
+    except Exception as e:
+        traceback.print_exc()
+        return [], str(e)
+    return hls_formats, None
+
 
 def _extract_playability_error(info, player_response, error_prefix=''):
     if info['formats']:
-- 
cgit v1.2.3