- Added support for loading cookies from a file.

The file must be a "# Netscape HTTP Cookie File"; same format as "hypervideo" requires.

The "cookies.txt" extension can be used for exporting the cookies from the browser.

This helps with the "429: Too Many Requests" issue.

See also: https://github.com/ytdl-org/hypervideo#how-do-i-pass-cookies-to-hypervideo

Signed-off-by: Jesús <heckyel@hyperbola.info>

1 files changed, 81 insertions, 37 deletions

diff --git a/lib/WWW/FairViewer.pm b/lib/WWW/FairViewer.pm
index 448206a..44bd1b2 100644
--- a/lib/WWW/FairViewer.pm
+++ b/lib/WWW/FairViewer.pm
@@ -41,21 +41,21 @@ my %valid_options = (
 
     # Main options
     v               => {valid => q[],                                                    default => 3},
-    page            => {valid => [qr/^(?!0+\z)\d+\z/],                                   default => 1},
-    http_proxy      => {valid => [qr{.}],                                                default => undef},
-    hl              => {valid => [qr/^\w+(?:[\-_]\w+)?\z/],                              default => undef},
+    page            => {valid => qr/^(?!0+\z)\d+\z/,                                     default => 1},
+    http_proxy      => {valid => qr/./,                                                  default => undef},
+    hl              => {valid => qr/^\w+(?:[\-_]\w+)?\z/,                                default => undef},
     maxResults      => {valid => [1 .. 50],                                              default => 10},
-    topicId         => {valid => [qr/^./],                                               default => undef},
+    topicId         => {valid => qr/./,                                                  default => undef},
     order           => {valid => [qw(relevance date rating viewCount title videoCount)], default => undef},
-    publishedAfter  => {valid => [qr/^\d+/],                                             default => undef},
-    publishedBefore => {valid => [qr/^\d+/],                                             default => undef},
-    channelId       => {valid => [qr/^[-\w]{2,}\z/],                                     default => undef},
+    publishedAfter  => {valid => qr/^\d+/,                                               default => undef},
+    publishedBefore => {valid => qr/^\d+/,                                               default => undef},
+    channelId       => {valid => qr/^[-\w]{2,}\z/,                                       default => undef},
     channelType     => {valid => [qw(any show)],                                         default => undef},
 
     # Video only options
     videoCaption    => {valid => [qw(any closedCaption none)],     default => undef},
     videoDefinition => {valid => [qw(any high standard)],          default => undef},
-    videoCategoryId => {valid => [qr/^\d+\z/],                     default => undef},
+    videoCategoryId => {valid => qr/^\d+\z/,                       default => undef},
     videoDimension  => {valid => [qw(any 2d 3d)],                  default => undef},
     videoDuration   => {valid => [qw(any short medium long)],      default => undef},
     videoEmbeddable => {valid => [qw(any true)],                   default => undef},
@@ -64,8 +64,8 @@ my %valid_options = (
     eventType       => {valid => [qw(completed live upcoming)],    default => undef},
     chart           => {valid => [qw(mostPopular)],                default => 'mostPopular'},
 
-    regionCode        => {valid => [qr/^[A-Z]{2}\z/i],         default => undef},
-    relevanceLanguage => {valid => [qr/^[a-z](?:\-\w+)?\z/i],  default => undef},
+    regionCode        => {valid => qr/^[A-Z]{2}\z/i,           default => undef},
+    relevanceLanguage => {valid => qr/^[a-z]+(?:\-\w+)?\z/i,   default => undef},
     safeSearch        => {valid => [qw(none moderate strict)], default => undef},
     videoType         => {valid => [qw(any episode movie)],    default => undef},
 
@@ -73,27 +73,28 @@ my %valid_options = (
     subscriptions_order => {valid => [qw(alphabetical relevance unread)], default => undef},
 
     # Misc
-    debug       => {valid => [0 .. 3],     default => 0},
-    lwp_timeout => {valid => [qr/^\d+\z/], default => 1},
-    config_dir  => {valid => [qr/^./],     default => q{.}},
-    cache_dir   => {valid => [qr/^./],     default => q{.}},
+    debug       => {valid => [0 .. 3],   default => 0},
+    timeout     => {valid => qr/^\d+\z/, default => 10},
+    config_dir  => {valid => qr/^./,     default => q{.}},
+    cache_dir   => {valid => qr/^./,     default => q{.}},
+    cookie_file => {valid => qr/^./,     default => undef},
 
     # Booleans
-    lwp_env_proxy => {valid => [1, 0], default => 1},
-    escape_utf8   => {valid => [1, 0], default => 0},
-    prefer_mp4    => {valid => [1, 0], default => 0},
-    prefer_av1    => {valid => [1, 0], default => 0},
+    env_proxy   => {valid => [1, 0], default => 1},
+    escape_utf8 => {valid => [1, 0], default => 0},
+    prefer_mp4  => {valid => [1, 0], default => 0},
+    prefer_av1  => {valid => [1, 0], default => 0},
 
     # API/OAuth
-    key           => {valid => [qr/^.{15}/], default => undef},
-    client_id     => {valid => [qr/^.{15}/], default => undef},
-    client_secret => {valid => [qr/^.{15}/], default => undef},
-    redirect_uri  => {valid => [qr/^.{15}/], default => undef},
-    access_token  => {valid => [qr/^.{15}/], default => undef},
-    refresh_token => {valid => [qr/^.{15}/], default => undef},
+    key           => {valid => qr/^.{15}/, default => undef},
+    client_id     => {valid => qr/^.{15}/, default => undef},
+    client_secret => {valid => qr/^.{15}/, default => undef},
+    redirect_uri  => {valid => qr/^.{15}/, default => undef},
+    access_token  => {valid => qr/^.{15}/, default => undef},
+    refresh_token => {valid => qr/^.{15}/, default => undef},
 
-    authentication_file => {valid => [qr/^./], default => undef},
-    api_host            => {valid => [qr/\w/], default => "auto"},
+    authentication_file => {valid => qr/^./, default => undef},
+    api_host            => {valid => qr/\w/, default => "auto"},
 
     # No input value allowed
     api_path         => {valid => q[], default => '/api/v1/'},
@@ -104,7 +105,7 @@ my %valid_options = (
 
 #<<<
     # LWP user agent
-    lwp_agent => {valid => [qr/^.{5}/], default => 'Mozilla/5.0 (Windows NT 10.0; Win64; gzip; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.0.0 Safari/537.36'},
+    user_agent => {valid => qr/^.{5}/, default => 'Mozilla/5.0 (Windows NT 10.0; Win64; gzip; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.0.0 Safari/537.36'},
 #>>>
 );
 
@@ -113,7 +114,7 @@ sub _our_smartmatch {
 
     $value // return 0;
 
-    if (ref($arg) eq '') {
+    if (not ref($arg)) {
         return ($value eq $arg);
     }
 
@@ -171,7 +172,7 @@ sub extra_video_info_fields {
 
     foreach my $key (keys %valid_options) {
 
-        if (ref $valid_options{$key}{valid} eq 'ARRAY') {
+        if (ref($valid_options{$key}{valid})) {
 
             # Create the 'set_*' subroutines
             *{__PACKAGE__ . '::set_' . $key} = sub {
@@ -274,10 +275,10 @@ sub set_lwp_useragent {
 
     $self->{lwp} = $lwp->new(
 
-        cookie_jar    => {},                       # temporary cookies
-        timeout       => $self->get_lwp_timeout,
+        cookie_jar    => {},                      # temporary cookies
+        timeout       => $self->get_timeout,
         show_progress => $self->get_debug,
-        agent         => $self->get_lwp_agent,
+        agent         => $self->get_user_agent,
 
         ssl_opts => {verify_hostname => 1},
 
@@ -306,7 +307,7 @@ sub set_lwp_useragent {
           )
         : (),
 
-        env_proxy => (defined($self->get_http_proxy) ? 0 : $self->get_lwp_env_proxy),
+        env_proxy => (defined($self->get_http_proxy) ? 0 : $self->get_env_proxy),
     );
 
     require LWP::ConnCache;
@@ -319,11 +320,42 @@ sub set_lwp_useragent {
     };
 
     my $agent = $self->{lwp};
-    $agent->ssl_opts(Timeout => 30);
+    $agent->ssl_opts(Timeout => $self->get_timeout);
     $agent->default_header('Accept-Encoding' => $accepted_encodings);
     $agent->conn_cache($cache);
     $agent->proxy(['http', 'https'], $self->get_http_proxy) if defined($self->get_http_proxy);
 
+    my $cookie_file = $self->get_cookie_file;
+
+    if (defined($cookie_file) and -f $cookie_file) {
+
+        if ($self->get_debug) {
+            say STDERR ":: Using cookies from: $cookie_file";
+        }
+
+        ## Netscape HTTP Cookies
+
+        # Chrome extension:
+        #   https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg
+
+        # Firefox extension:
+        #   https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/
+
+        # See also:
+        #   https://github.com/ytdl-org/hypervideo#how-do-i-pass-cookies-to-hypervideo
+
+        require HTTP::Cookies::Netscape;
+
+        my $cookies = HTTP::Cookies::Netscape->new(
+                                                   hide_cookie2 => 1,
+                                                   autosave     => 1,
+                                                   file         => $cookie_file,
+                                                  );
+
+        $cookies->load;
+        $agent->cookie_jar($cookies);
+    }
+
     push @{$self->{lwp}->requests_redirectable}, 'POST';
     return $self->{lwp};
 }
@@ -505,7 +537,7 @@ sub get_invidious_instances {
 
         require LWP::UserAgent;
 
-        my $lwp = LWP::UserAgent->new(timeout => 10);
+        my $lwp = LWP::UserAgent->new(timeout => $self->get_timeout);
         $lwp->show_progress(1) if $self->get_debug;
         my $resp = $lwp->get("https://instances.invidio.us/instances.json");
 
@@ -710,8 +742,15 @@ sub _extract_from_ytdl {
 
     $self->_ytdl_is_available() || return;
 
-    my $json = $self->proxy_stdout('hypervideo', '--all-formats', '--dump-single-json',
-                                   quotemeta("https://www.youtube.com/watch?v=" . $videoID));
+    my @ytdl_cmd = ('hypervideo', '--all-formats', '--dump-single-json');
+
+    my $cookie_file = $self->get_cookie_file;
+
+    if (defined($cookie_file) and -f $cookie_file) {
+        push @ytdl_cmd, '--cookies', $cookie_file;
+    }
+
+    my $json = $self->proxy_stdout(@ytdl_cmd, quotemeta("https://www.youtube.com/watch?v=" . $videoID));
 
     my $ref = $self->parse_json_string($json);
 
@@ -1010,6 +1049,11 @@ sub get_streaming_urls {
         }
     }
 
+    if ($self->get_debug) {
+        my $count = scalar(@streaming_urls);
+        say STDERR ":: Found $count streaming URLs...";
+    }
+
     # Try again with hypervideo
     if (!@streaming_urls or $info{status} =~ /fail|error/i) {
         @streaming_urls = $self->_fallback_extract_urls($videoID);