upstream

1 files changed, 297 insertions, 54 deletions

diff --git a/lib/WWW/FairViewer/Utils.pm b/lib/WWW/FairViewer/Utils.pm
index 8cdcce3..1076af2 100644
--- a/lib/WWW/FairViewer/Utils.pm
+++ b/lib/WWW/FairViewer/Utils.pm
@@ -96,6 +96,9 @@ Returns time from seconds.
 
 sub format_time {
     my ($self, $sec) = @_;
+
+    $sec //= 0;
+
     $sec >= 3600
       ? join q{:}, map { sprintf '%02d', $_ } $sec / 3600 % 24, $sec / 60 % 60, $sec % 60
       : join q{:}, map { sprintf '%02d', $_ } $sec / 60 % 60, $sec % 60;
@@ -133,6 +136,8 @@ Return string "04 May 2010" from "2010-05-04T00:25:55.000Z"
 sub format_date {
     my ($self, $date) = @_;
 
+    $date // return undef;
+
     # 2010-05-04T00:25:55.000Z
     # to: 04 May 2010
 
@@ -158,6 +163,8 @@ Return the (approximated) age for a given date of the form "2010-05-04T00:25:55.
 sub date_to_age {
     my ($self, $date) = @_;
 
+    $date // return undef;
+
     $date =~ m{^
         (?<year>\d{4})
            -
@@ -177,6 +184,21 @@ sub date_to_age {
     $year  += 1900;
     $month += 1;
 
+    my %month_days = (
+                      1  => 31,
+                      2  => 28,
+                      3  => 31,
+                      4  => 30,
+                      5  => 31,
+                      6  => 30,
+                      7  => 31,
+                      8  => 31,
+                      9  => 30,
+                      10 => 31,
+                      11 => 30,
+                      12 => 31,
+                     );
+
     my $lambda = sub {
 
         if ($year == $+{year}) {
@@ -192,6 +214,14 @@ sub date_to_age {
                 }
                 return join(' ', $day - $+{day}, 'days');
             }
+
+            if ($month - $+{month} == 1) {
+                my $day_diff = $+{day} - $day;
+                if ($day_diff > 0 and $day_diff < $month_days{$+{month} + 0}) {
+                    return join(' ', $month_days{$+{month} + 0} - $day_diff, 'days');
+                }
+            }
+
             return join(' ', $month - $+{month}, 'months');
         }
 
@@ -227,7 +257,7 @@ sub has_entries {
 
     if (ref($result->{results}) eq 'HASH') {
 
-        foreach my $type (qw(comments videos playlists)) {
+        foreach my $type (qw(comments videos playlists entries)) {
             if (exists $result->{results}{$type}) {
                 ref($result->{results}{$type}) eq 'ARRAY' or return 0;
                 return (@{$result->{results}{$type}} > 0);
@@ -252,15 +282,21 @@ sub has_entries {
     return 1;    # maybe?
 }
 
-=head2 normalize_video_title($title, $fat32safe)
+=head2 normalize_filename($title, $fat32safe)
 
 Replace file-unsafe characters and trim spaces.
 
 =cut
 
-sub normalize_video_title {
+sub normalize_filename {
     my ($self, $title, $fat32safe) = @_;
 
+    state $unix_like = $^O =~ /^(?:linux|freebsd|openbsd)\z/i;
+
+    if (not $fat32safe and not $unix_like) {
+        $fat32safe = 1;
+    }
+
     if ($fat32safe) {
         $title =~ s/: / - /g;
         $title =~ tr{:"*/?\\|}{;'+%!%%};    # "
@@ -270,7 +306,9 @@ sub normalize_video_title {
         $title =~ tr{/}{%};
     }
 
-    join(q{ }, split(q{ }, $title));
+    my $basename = join(q{ }, split(q{ }, $title));
+    $basename = substr($basename, 0, 200);    # make sure the filename is not too long
+    return $basename;
 }
 
 =head2 format_text(%opt)
@@ -299,20 +337,32 @@ sub format_text {
     my $fat32safe = $opt{fat32safe};
 
     my %special_tokens = (
-        ID          => sub { $self->get_video_id($info) },
-        AUTHOR      => sub { $self->get_channel_title($info) },
-        CHANNELID   => sub { $self->get_channel_id($info) },
-        DEFINITION  => sub { $self->get_definition($info) },
-        DIMENSION   => sub { $self->get_dimension($info) },
+        ID         => sub { $self->get_video_id($info) },
+        AUTHOR     => sub { $self->get_channel_title($info) },
+        CHANNELID  => sub { $self->get_channel_id($info) },
+        DEFINITION => sub { $self->get_definition($info) },
+        DIMENSION  => sub { $self->get_dimension($info) },
+
         VIEWS       => sub { $self->get_views($info) },
         VIEWS_SHORT => sub { $self->get_views_approx($info) },
-        LIKES       => sub { $self->get_likes($info) },
-        DISLIKES    => sub { $self->get_dislikes($info) },
+
+        VIDEOS       => sub { $self->set_thousands($self->get_channel_video_count($info)) },
+        VIDEOS_SHORT => sub { $self->short_human_number($self->get_channel_video_count($info)) },
+
+        SUBS       => sub { $self->get_channel_subscriber_count($info) },
+        SUBS_SHORT => sub { $self->short_human_number($self->get_channel_subscriber_count($info)) },
+
+        ITEMS       => sub { $self->set_thousands($self->get_playlist_item_count($info)) },
+        ITEMS_SHORT => sub { $self->short_human_number($self->get_playlist_item_count($info)) },
+
+        LIKES    => sub { $self->get_likes($info) },
+        DISLIKES => sub { $self->get_dislikes($info) },
+
         COMMENTS    => sub { $self->get_comments($info) },
         DURATION    => sub { $self->get_duration($info) },
         TIME        => sub { $self->get_time($info) },
         TITLE       => sub { $self->get_title($info) },
-        FTITLE      => sub { $self->normalize_video_title($self->get_title($info), $fat32safe) },
+        FTITLE      => sub { $self->normalize_filename($self->get_title($info), $fat32safe) },
         CAPTION     => sub { $self->get_caption($info) },
         PUBLISHED   => sub { $self->get_publication_date($info) },
         AGE         => sub { $self->get_publication_age($info) },
@@ -386,8 +436,8 @@ sub format_text {
     $text =~ s/$escapes_re/$special_escapes{$1}/g;
 
     $escape
-      ? $text =~ s/$tokens_re/\Q${\$special_tokens{$1}()}\E/gr
-      : $text =~ s/$tokens_re/${\$special_tokens{$1}()}/gr;
+      ? $text =~ s<$tokens_re><\Q${\($special_tokens{$1}() // '')}\E>gr
+      : $text =~ s<$tokens_re><${\($special_tokens{$1}() // '')}>gr;
 }
 
 =head2 set_thousands($num)
@@ -487,13 +537,112 @@ sub get_description {
     $desc = HTML::Entities::decode_entities($desc);
     $desc =~ s/^\s+//;
 
-    if (not $desc =~ /\S/) {
+    if (not $desc =~ /\S/ or length($desc) < length($info->{description} // '')) {
         $desc = $info->{description} // '';
     }
 
     ($desc =~ /\S/) ? $desc : 'No description available...';
 }
 
+sub read_lines_from_file {
+    my ($self, $file, $mode) = @_;
+
+    $mode //= '<';
+
+    open(my $fh, $mode, $file) or return;
+    chomp(my @lines = <$fh>);
+    close $fh;
+
+    my %seen;
+
+    # Keep the most recent ones
+    @lines = reverse(@lines);
+    @lines = grep { !$seen{$_}++ } @lines;
+
+    return @lines;
+}
+
+sub read_channels_from_file {
+    my ($self, $file, $mode) = @_;
+
+    $mode //= '<:utf8';
+
+    # Read channels and remove duplicates
+    my %channels = map { split(/ /, $_, 2) } $self->read_lines_from_file($file, $mode);
+
+    # Filter valid channels and pair with channel ID with title
+    my @channels = map { [$_, $channels{$_}] } grep { defined($channels{$_}) } keys %channels;
+
+    # Sort channels by channel name
+    @channels = sort { CORE::fc($a->[1]) cmp CORE::fc($b->[1]) } @channels;
+
+    return @channels;
+}
+
+sub get_local_playlist_filenames {
+    my ($self, $dir) = @_;
+    require Encode;
+    grep { -f $_ } sort { CORE::fc($a) cmp CORE::fc($b) } map { Encode::decode_utf8($_) } glob("$dir/*.dat");
+}
+
+sub make_local_playlist_filename {
+    my ($self, $title, $playlistID) = @_;
+    my $basename = $title . ' -- ' . $playlistID . '.txt';
+    $basename = $self->normalize_filename($basename);
+    return $basename;
+}
+
+sub local_playlist_snippet {
+    my ($self, $id) = @_;
+
+    require File::Basename;
+    my $title = File::Basename::basename($id);
+
+    $title =~ s/\.dat\z//;
+    $title =~ s/ -- PL[-\w]+\z//;
+    $title =~ s/_/ /g;
+    $title = ucfirst($title);
+
+    require Storable;
+    my $entries = eval { Storable::retrieve($id) } // [];
+
+    if (ref($entries) ne 'ARRAY') {
+        $entries = [];
+    }
+
+    my $video_count = 0;
+    my $video_id    = undef;
+
+    if (@$entries) {
+        $video_id    = $self->get_video_id($entries->[0]);
+        $video_count = scalar(@$entries);
+    }
+
+    scalar {
+            author            => "local",
+            authorId          => "local",
+            description       => $title,
+            playlistId        => $id,
+            playlistThumbnail => (defined($video_id) ? "https://i.ytimg.com/vi/$video_id/mqdefault.jpg" : undef),
+            title             => $title,
+            type              => "playlist",
+            videoCount        => $video_count,
+           };
+}
+
+sub local_channel_snippet {
+    my ($self, $id, $title) = @_;
+
+    scalar {
+            author      => $title,
+            authorId    => $id,
+            type        => "channel",
+            description => "<local channel>",
+            subCount    => undef,
+            videoCount  => undef,
+           };
+}
+
 =head2 get_title($info)
 
 Get title.
@@ -545,7 +694,7 @@ sub get_thumbnail_url {
         $url = eval { $wanted[0]{url} } // return '';
     }
     else {
-        warn "[!] Couldn't find thumbnail of type <<$type>>...";
+        ## warn "[!] Couldn't find thumbnail of type <<$type>>...";
         $url = eval { $thumbs[0]{url} } // return '';
     }
 
@@ -559,7 +708,7 @@ sub get_channel_title {
     my ($self, $info) = @_;
 
     #$info->{snippet}{channelTitle} || $self->get_channel_id($info);
-    $info->{author};
+    $info->{author} // $info->{title};
 }
 
 sub get_author {
@@ -572,6 +721,31 @@ sub get_comment_id {
     $info->{commentId};
 }
 
+sub get_video_count {
+    my ($self, $info) = @_;
+    $info->{videoCount} // 0;
+}
+
+sub get_subscriber_count {
+    my ($self, $info) = @_;
+    $info->{subCount} // 0;
+}
+
+sub get_channel_subscriber_count {
+    my ($self, $info) = @_;
+    $info->{subCount} // 0;
+}
+
+sub get_channel_video_count {
+    my ($self, $info) = @_;
+    $info->{videoCount} // 0;
+}
+
+sub get_playlist_item_count {
+    my ($self, $info) = @_;
+    $info->{videoCount} // 0;
+}
+
 sub get_comment_content {
     my ($self, $info) = @_;
     $info->{content};
@@ -579,24 +753,23 @@ sub get_comment_content {
 
 sub get_id {
     my ($self, $info) = @_;
-
-    #$info->{id};
     $info->{videoId};
 }
 
-sub get_channel_id {
+sub get_rating {
     my ($self, $info) = @_;
+    my $rating = $info->{rating} // return;
+    sprintf('%.2f', $rating);
+}
 
-    #$info->{snippet}{resourceId}{channelId} // $info->{snippet}{channelId};
+sub get_channel_id {
+    my ($self, $info) = @_;
     $info->{authorId};
 }
 
 sub get_category_id {
     my ($self, $info) = @_;
-
-    #$info->{snippet}{resourceId}{categoryId} // $info->{snippet}{categoryId};
-    #"unknown";
-    $info->{genre} // 'Unknown';
+    $info->{genre} // $info->{category} // 'Unknown';
 }
 
 sub get_category_name {
@@ -620,9 +793,7 @@ sub get_category_name {
                          29 => 'Nonprofits & Activism',
                         };
 
-    #$categories->{$self->get_category_id($info) // ''} // 'Unknown';
-
-    $info->{genre} // 'Unknown';
+    $info->{genre} // $info->{category} // 'Unknown';
 }
 
 sub get_publication_date {
@@ -635,8 +806,80 @@ sub get_publication_date {
     require Encode;
     require Time::Piece;
 
-    my $time = Time::Piece->new($info->{published});
-    Encode::decode_utf8($time->strftime("%d %B %Y"));
+    my $time;
+
+    if (defined($info->{published})) {
+        $time = eval { Time::Piece->new($info->{published}) };
+    }
+    elsif (defined($info->{publishDate})) {
+        if ($info->{publishDate} =~ /^[0-9]+\z/) {    # time given as "%yyyy%mm%dd" (from hypervideo)
+            $time = eval { Time::Piece->strptime($info->{publishDate}, '%Y%m%d') };
+        }
+        else {
+            $time = eval { Time::Piece->strptime($info->{publishDate}, '%Y-%m-%d') };
+        }
+    }
+
+    defined($time) ? Encode::decode_utf8($time->strftime("%d %B %Y")) : undef;
+}
+
+sub get_publication_time {
+    my ($self, $info) = @_;
+
+    require Time::Piece;
+    require Time::Seconds;
+
+    if ($self->get_time($info) eq 'LIVE') {
+        my $time = $info->{timestamp} // Time::Piece->new();
+
+        if (ref($time) eq 'ARRAY') {
+            $time = bless($time, "Time::Piece");
+        }
+
+        return $time;
+    }
+
+    if (defined($info->{publishedText})) {
+
+        my $age = $info->{publishedText};
+        my $t   = $info->{timestamp} // Time::Piece->new();
+
+        if (ref($t) eq 'ARRAY') {
+            $t = bless($t, "Time::Piece");
+        }
+
+        if ($age =~ /^(\d+) sec/) {
+            $t -= $1;
+        }
+
+        if ($age =~ /^(\d+) min/) {
+            $t -= $1 * Time::Seconds::ONE_MINUTE();
+        }
+
+        if ($age =~ /^(\d+) hour/) {
+            $t -= $1 * Time::Seconds::ONE_HOUR();
+        }
+
+        if ($age =~ /^(\d+) day/) {
+            $t -= $1 * Time::Seconds::ONE_DAY();
+        }
+
+        if ($age =~ /^(\d+) week/) {
+            $t -= $1 * Time::Seconds::ONE_WEEK();
+        }
+
+        if ($age =~ /^(\d+) month/) {
+            $t -= $1 * Time::Seconds::ONE_MONTH();
+        }
+
+        if ($age =~ /^(\d+) year/) {
+            $t -= $1 * Time::Seconds::ONE_YEAR();
+        }
+
+        return $t;
+    }
+
+    return $self->get_publication_date($info);    # should not happen
 }
 
 sub get_publication_age {
@@ -674,22 +917,17 @@ sub get_publication_age_approx {
 
 sub get_duration {
     my ($self, $info) = @_;
-
-    #$self->format_duration($info->{contentDetails}{duration});
-    #$self->format_duration($info->{lengthSeconds});
     $info->{lengthSeconds};
 }
 
 sub get_time {
     my ($self, $info) = @_;
 
-    if ($info->{liveNow}) {
+    if ($info->{liveNow} and ($self->get_duration($info) || 0) == 0) {
         return 'LIVE';
     }
 
     $self->format_time($self->get_duration($info));
-
-    #$self->format_time($self->get_duration($info));
 }
 
 sub get_definition {
@@ -721,39 +959,44 @@ sub get_views {
     $info->{viewCount} // 0;
 }
 
-sub get_views_approx {
-    my ($self, $info) = @_;
-    my $views = $self->get_views($info);
+sub short_human_number {
+    my ($self, $int) = @_;
 
-    if ($views < 1000) {
-        return $views;
+    if ($int < 1000) {
+        return $int;
     }
 
-    if ($views >= 10 * 1e9) {    # ten billions
-        return sprintf("%dB", int($views / 1e9));
+    if ($int >= 10 * 1e9) {    # ten billions
+        return sprintf("%dB", int($int / 1e9));
     }
 
-    if ($views >= 1e9) {         # billions
-        return sprintf("%.2gB", $views / 1e9);
+    if ($int >= 1e9) {         # billions
+        return sprintf("%.2gB", $int / 1e9);
     }
 
-    if ($views >= 10 * 1e6) {    # ten millions
-        return sprintf("%dM", int($views / 1e6));
+    if ($int >= 10 * 1e6) {    # ten millions
+        return sprintf("%dM", int($int / 1e6));
     }
 
-    if ($views >= 1e6) {         # millions
-        return sprintf("%.2gM", $views / 1e6);
+    if ($int >= 1e6) {         # millions
+        return sprintf("%.2gM", $int / 1e6);
     }
 
-    if ($views >= 10 * 1e3) {    # ten thousands
-        return sprintf("%dK", int($views / 1e3));
+    if ($int >= 10 * 1e3) {    # ten thousands
+        return sprintf("%dK", int($int / 1e3));
     }
 
-    if ($views >= 1e3) {         # thousands
-        return sprintf("%.2gK", $views / 1e3);
+    if ($int >= 1e3) {         # thousands
+        return sprintf("%.2gK", $int / 1e3);
     }
 
-    return $views;
+    return $int;
+}
+
+sub get_views_approx {
+    my ($self, $info) = @_;
+    my $views = $self->get_views($info);
+    $self->short_human_number($views);
 }
 
 sub get_likes {