- Added support for loading cookies from a file.

The file must be a "# Netscape HTTP Cookie File"; same format as "hypervideo" requires.

The "cookies.txt" extension can be used for exporting the cookies from the browser.

This helps with the "429: Too Many Requests" issue.

See also: https://github.com/ytdl-org/hypervideo#how-do-i-pass-cookies-to-hypervideo

Signed-off-by: Jesús <heckyel@hyperbola.info>

3 files changed, 112 insertions, 51 deletions

diff --git a/bin/fair-viewer b/bin/fair-viewer
index 6907290..b1a9c4b 100755
--- a/bin/fair-viewer
+++ b/bin/fair-viewer
@@ -16,7 +16,7 @@
 #-------------------------------------------------------
 #  fair-viewer
 #  Fork: 14 February 2020
-#  Edit: 06 September 2020
+#  Edit: 09 September 2020
 #  https://framagit.org/heckyel/fair-viewer
 #-------------------------------------------------------
 
@@ -218,6 +218,9 @@ my %CONFIG = (
     # Others
     autoplay_mode        => 0,
     http_proxy           => undef,
+    cookie_file          => undef,
+    user_agent           => undef,
+    timeout              => undef,
     env_proxy            => 1,
     confirm              => 0,
     debug                => 0,
@@ -602,13 +605,16 @@ my $yv_obj = WWW::FairViewer->new(
                                      escape_utf8         => 1,
                                      config_dir          => $config_dir,
                                      cache_dir           => $opt{cache_dir},
-                                     lwp_env_proxy       => $opt{env_proxy},
-                                     authentication_file => $authentication_file,
-                                    );
+                                     env_proxy           => $opt{env_proxy},
+                                     cookie_file         => $opt{cookie_file},
+                                     http_proxy          => $opt{http_proxy},
+                                     user_agent          => $opt{user_agent},
+                                     timeout             => $opt{timeout},
+                                );
 
 require WWW::FairViewer::Utils;
 my $yv_utils = WWW::FairViewer::Utils->new(youtube_url_format => $opt{youtube_video_url},
-                                              thousand_separator => $opt{thousand_separator},);
+                                            thousand_separator => $opt{thousand_separator},);
 
 {    # Apply the configuration file
     my %temp = %CONFIG;
@@ -793,6 +799,8 @@ usage: $execname [options] ([url] | [keywords])
  * Other
    --api=s              : set an API host from https://instances.invidio.us/
    --api=auto           : use a random instance of invidious
+   --cookies=s          : file to read cookies from and dump cookie
+   --user-agent=s       : specify a custom user agent
    --proxy=s            : set HTTP(S)/SOCKS proxy: 'proto://domain.tld:port/'
                           If authentication required,
                           use 'proto://user:pass\@domain.tld:port/'
@@ -1094,7 +1102,8 @@ sub apply_configuration {
                              publishedAfter publishedBefore
                              safeSearch regionCode debug hl
                              http_proxy page comments_order
-                             subscriptions_order
+                             subscriptions_order user_agent
+                             cookie_file timeout
                              )
       ) {
 
@@ -1515,7 +1524,9 @@ sub parse_arguments {
         'related-videos|rv=s'         => \$opt{related_videos},
         'popular-videos|popular|pv=s' => \$opt{popular_videos},
 
-        'http_proxy|http-proxy|proxy=s' => \$opt{http_proxy},
+        'cookie-file|cookies=s'          => \$opt{cookie_file},
+        'user-agent|agent=s'             => \$opt{user_agent},
+        'http-proxy|https-proxy|proxy=s' => \$opt{http_proxy},
 
         'catlang|cl|hl=s'        => \$opt{hl},
         'category|cat-id|cat=s'  => \$opt{category_id},
diff --git a/bin/gtk-fair-viewer b/bin/gtk-fair-viewer
index 4020fd3..fadbf18 100755
--- a/bin/gtk-fair-viewer
+++ b/bin/gtk-fair-viewer
@@ -16,7 +16,7 @@
 #-------------------------------------------------------
 #  GTK Fair Viewer
 #  Fork: 14 February 2020
-#  Edit: 06 September 2020
+#  Edit: 09 September 2020
 #  https://framagit.org/heckyel/fair-viewer
 #-------------------------------------------------------
 
@@ -222,6 +222,9 @@ my %CONFIG = (
     # Others
     env_proxy   => 1,
     http_proxy  => undef,
+    timeout     => undef,
+    user_agent  => undef,
+    cookie_file => undef,
     prefer_fork => (($^O eq 'linux') ? 0 : 1),
     debug       => 0,
     fullscreen  => 0,
@@ -808,12 +811,14 @@ my $yv_obj = WWW::FairViewer->new(
                                      escape_utf8         => 1,
                                      config_dir          => $config_dir,
                                      hl                  => $CONFIG{hl},
-                                     lwp_env_proxy       => $CONFIG{env_proxy},
+                                     env_proxy           => $CONFIG{env_proxy},
                                      cache_dir           => $CONFIG{cache_dir},
-                                     authentication_file => $authentication_file,
-                                    );
+                                     cookie_file         => $CONFIG{cookie_file},
+                                     user_agent          => $CONFIG{user_agent},
+                                     timeout             => $CONFIG{timeout},
+                                );
 
-$yv_obj->load_authentication_tokens();
+#$yv_obj->load_authentication_tokens();
 
 if (defined $yv_obj->get_access_token()) {
     show_user_panel();
@@ -824,7 +829,7 @@ else {
 
 require WWW::FairViewer::Utils;
 my $yv_utils = WWW::FairViewer::Utils->new(thousand_separator => $CONFIG{thousand_separator},
-                                              youtube_url_format => $CONFIG{youtube_video_url},);
+                                            youtube_url_format => $CONFIG{youtube_video_url},);
 
 # Set default combobox values
 $definition_combobox->set_active(0);
@@ -864,7 +869,8 @@ sub apply_configuration {
                              videoEmbeddable videoLicense
                              publishedAfter publishedBefore
                              regionCode videoCategoryId
-                             debug http_proxy
+                             debug http_proxy user_agent
+                             timeout cookie_file
                              )
       ) {
 
diff --git a/lib/WWW/FairViewer.pm b/lib/WWW/FairViewer.pm
index 448206a..44bd1b2 100644
--- a/lib/WWW/FairViewer.pm
+++ b/lib/WWW/FairViewer.pm
@@ -41,21 +41,21 @@ my %valid_options = (
 
     # Main options
     v               => {valid => q[],                                                    default => 3},
-    page            => {valid => [qr/^(?!0+\z)\d+\z/],                                   default => 1},
-    http_proxy      => {valid => [qr{.}],                                                default => undef},
-    hl              => {valid => [qr/^\w+(?:[\-_]\w+)?\z/],                              default => undef},
+    page            => {valid => qr/^(?!0+\z)\d+\z/,                                     default => 1},
+    http_proxy      => {valid => qr/./,                                                  default => undef},
+    hl              => {valid => qr/^\w+(?:[\-_]\w+)?\z/,                                default => undef},
     maxResults      => {valid => [1 .. 50],                                              default => 10},
-    topicId         => {valid => [qr/^./],                                               default => undef},
+    topicId         => {valid => qr/./,                                                  default => undef},
     order           => {valid => [qw(relevance date rating viewCount title videoCount)], default => undef},
-    publishedAfter  => {valid => [qr/^\d+/],                                             default => undef},
-    publishedBefore => {valid => [qr/^\d+/],                                             default => undef},
-    channelId       => {valid => [qr/^[-\w]{2,}\z/],                                     default => undef},
+    publishedAfter  => {valid => qr/^\d+/,                                               default => undef},
+    publishedBefore => {valid => qr/^\d+/,                                               default => undef},
+    channelId       => {valid => qr/^[-\w]{2,}\z/,                                       default => undef},
     channelType     => {valid => [qw(any show)],                                         default => undef},
 
     # Video only options
     videoCaption    => {valid => [qw(any closedCaption none)],     default => undef},
     videoDefinition => {valid => [qw(any high standard)],          default => undef},
-    videoCategoryId => {valid => [qr/^\d+\z/],                     default => undef},
+    videoCategoryId => {valid => qr/^\d+\z/,                       default => undef},
     videoDimension  => {valid => [qw(any 2d 3d)],                  default => undef},
     videoDuration   => {valid => [qw(any short medium long)],      default => undef},
     videoEmbeddable => {valid => [qw(any true)],                   default => undef},
@@ -64,8 +64,8 @@ my %valid_options = (
     eventType       => {valid => [qw(completed live upcoming)],    default => undef},
     chart           => {valid => [qw(mostPopular)],                default => 'mostPopular'},
 
-    regionCode        => {valid => [qr/^[A-Z]{2}\z/i],         default => undef},
-    relevanceLanguage => {valid => [qr/^[a-z](?:\-\w+)?\z/i],  default => undef},
+    regionCode        => {valid => qr/^[A-Z]{2}\z/i,           default => undef},
+    relevanceLanguage => {valid => qr/^[a-z]+(?:\-\w+)?\z/i,   default => undef},
     safeSearch        => {valid => [qw(none moderate strict)], default => undef},
     videoType         => {valid => [qw(any episode movie)],    default => undef},
 
@@ -73,27 +73,28 @@ my %valid_options = (
     subscriptions_order => {valid => [qw(alphabetical relevance unread)], default => undef},
 
     # Misc
-    debug       => {valid => [0 .. 3],     default => 0},
-    lwp_timeout => {valid => [qr/^\d+\z/], default => 1},
-    config_dir  => {valid => [qr/^./],     default => q{.}},
-    cache_dir   => {valid => [qr/^./],     default => q{.}},
+    debug       => {valid => [0 .. 3],   default => 0},
+    timeout     => {valid => qr/^\d+\z/, default => 10},
+    config_dir  => {valid => qr/^./,     default => q{.}},
+    cache_dir   => {valid => qr/^./,     default => q{.}},
+    cookie_file => {valid => qr/^./,     default => undef},
 
     # Booleans
-    lwp_env_proxy => {valid => [1, 0], default => 1},
-    escape_utf8   => {valid => [1, 0], default => 0},
-    prefer_mp4    => {valid => [1, 0], default => 0},
-    prefer_av1    => {valid => [1, 0], default => 0},
+    env_proxy   => {valid => [1, 0], default => 1},
+    escape_utf8 => {valid => [1, 0], default => 0},
+    prefer_mp4  => {valid => [1, 0], default => 0},
+    prefer_av1  => {valid => [1, 0], default => 0},
 
     # API/OAuth
-    key           => {valid => [qr/^.{15}/], default => undef},
-    client_id     => {valid => [qr/^.{15}/], default => undef},
-    client_secret => {valid => [qr/^.{15}/], default => undef},
-    redirect_uri  => {valid => [qr/^.{15}/], default => undef},
-    access_token  => {valid => [qr/^.{15}/], default => undef},
-    refresh_token => {valid => [qr/^.{15}/], default => undef},
+    key           => {valid => qr/^.{15}/, default => undef},
+    client_id     => {valid => qr/^.{15}/, default => undef},
+    client_secret => {valid => qr/^.{15}/, default => undef},
+    redirect_uri  => {valid => qr/^.{15}/, default => undef},
+    access_token  => {valid => qr/^.{15}/, default => undef},
+    refresh_token => {valid => qr/^.{15}/, default => undef},
 
-    authentication_file => {valid => [qr/^./], default => undef},
-    api_host            => {valid => [qr/\w/], default => "auto"},
+    authentication_file => {valid => qr/^./, default => undef},
+    api_host            => {valid => qr/\w/, default => "auto"},
 
     # No input value allowed
     api_path         => {valid => q[], default => '/api/v1/'},
@@ -104,7 +105,7 @@ my %valid_options = (
 
 #<<<
     # LWP user agent
-    lwp_agent => {valid => [qr/^.{5}/], default => 'Mozilla/5.0 (Windows NT 10.0; Win64; gzip; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.0.0 Safari/537.36'},
+    user_agent => {valid => qr/^.{5}/, default => 'Mozilla/5.0 (Windows NT 10.0; Win64; gzip; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.0.0 Safari/537.36'},
 #>>>
 );
 
@@ -113,7 +114,7 @@ sub _our_smartmatch {
 
     $value // return 0;
 
-    if (ref($arg) eq '') {
+    if (not ref($arg)) {
         return ($value eq $arg);
     }
 
@@ -171,7 +172,7 @@ sub extra_video_info_fields {
 
     foreach my $key (keys %valid_options) {
 
-        if (ref $valid_options{$key}{valid} eq 'ARRAY') {
+        if (ref($valid_options{$key}{valid})) {
 
             # Create the 'set_*' subroutines
             *{__PACKAGE__ . '::set_' . $key} = sub {
@@ -274,10 +275,10 @@ sub set_lwp_useragent {
 
     $self->{lwp} = $lwp->new(
 
-        cookie_jar    => {},                       # temporary cookies
-        timeout       => $self->get_lwp_timeout,
+        cookie_jar    => {},                      # temporary cookies
+        timeout       => $self->get_timeout,
         show_progress => $self->get_debug,
-        agent         => $self->get_lwp_agent,
+        agent         => $self->get_user_agent,
 
         ssl_opts => {verify_hostname => 1},
 
@@ -306,7 +307,7 @@ sub set_lwp_useragent {
           )
         : (),
 
-        env_proxy => (defined($self->get_http_proxy) ? 0 : $self->get_lwp_env_proxy),
+        env_proxy => (defined($self->get_http_proxy) ? 0 : $self->get_env_proxy),
     );
 
     require LWP::ConnCache;
@@ -319,11 +320,42 @@ sub set_lwp_useragent {
     };
 
     my $agent = $self->{lwp};
-    $agent->ssl_opts(Timeout => 30);
+    $agent->ssl_opts(Timeout => $self->get_timeout);
     $agent->default_header('Accept-Encoding' => $accepted_encodings);
     $agent->conn_cache($cache);
     $agent->proxy(['http', 'https'], $self->get_http_proxy) if defined($self->get_http_proxy);
 
+    my $cookie_file = $self->get_cookie_file;
+
+    if (defined($cookie_file) and -f $cookie_file) {
+
+        if ($self->get_debug) {
+            say STDERR ":: Using cookies from: $cookie_file";
+        }
+
+        ## Netscape HTTP Cookies
+
+        # Chrome extension:
+        #   https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg
+
+        # Firefox extension:
+        #   https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/
+
+        # See also:
+        #   https://github.com/ytdl-org/hypervideo#how-do-i-pass-cookies-to-hypervideo
+
+        require HTTP::Cookies::Netscape;
+
+        my $cookies = HTTP::Cookies::Netscape->new(
+                                                   hide_cookie2 => 1,
+                                                   autosave     => 1,
+                                                   file         => $cookie_file,
+                                                  );
+
+        $cookies->load;
+        $agent->cookie_jar($cookies);
+    }
+
     push @{$self->{lwp}->requests_redirectable}, 'POST';
     return $self->{lwp};
 }
@@ -505,7 +537,7 @@ sub get_invidious_instances {
 
         require LWP::UserAgent;
 
-        my $lwp = LWP::UserAgent->new(timeout => 10);
+        my $lwp = LWP::UserAgent->new(timeout => $self->get_timeout);
         $lwp->show_progress(1) if $self->get_debug;
         my $resp = $lwp->get("https://instances.invidio.us/instances.json");
 
@@ -710,8 +742,15 @@ sub _extract_from_ytdl {
 
     $self->_ytdl_is_available() || return;
 
-    my $json = $self->proxy_stdout('hypervideo', '--all-formats', '--dump-single-json',
-                                   quotemeta("https://www.youtube.com/watch?v=" . $videoID));
+    my @ytdl_cmd = ('hypervideo', '--all-formats', '--dump-single-json');
+
+    my $cookie_file = $self->get_cookie_file;
+
+    if (defined($cookie_file) and -f $cookie_file) {
+        push @ytdl_cmd, '--cookies', $cookie_file;
+    }
+
+    my $json = $self->proxy_stdout(@ytdl_cmd, quotemeta("https://www.youtube.com/watch?v=" . $videoID));
 
     my $ref = $self->parse_json_string($json);
 
@@ -1010,6 +1049,11 @@ sub get_streaming_urls {
         }
     }
 
+    if ($self->get_debug) {
+        my $count = scalar(@streaming_urls);
+        say STDERR ":: Found $count streaming URLs...";
+    }
+
     # Try again with hypervideo
     if (!@streaming_urls or $info{status} =~ /fail|error/i) {
         @streaming_urls = $self->_fallback_extract_urls($videoID);