#!/usr/bin/perl -w

use File::Basename;
use JSON qw( decode_json );
use Data::Dumper;
use utf8;
use open qw( :std :encoding(UTF-8) );
use Encode qw( decode encode_utf8 );
use URI::Escape;
use HTML::Entities;

binmode STDOUT, ':utf8';

# Database
my @db_list = ();

# Command line parameter
my $host = '';
# Type of list
my $b_browse = 1; # default
my $b_watchlist = 0;

my $b_get_json = 0;
my $b_resume = 0;
my $config_file = 'crdl.conf';
my $timestamp = '';
my $help = 1;
while (my $opt = shift) {
    $help = 0;
    if ($opt=~/^-/) {
        $help = 1 if ($opt=~/^-help/i);
        $config_file = shift if ($opt=~/^-conf/i);
        if ($opt=~/^-timestamp/i) {
            $timestamp = shift;
            #print "Using timstamp = ${timestamp}\n";
        }
        $b_get_json = 1 if ($opt=~/^-json/i);
        $b_resume = 1 if ($opt=~/^-resume/i);
    } elsif ($opt=~/^watchlist$/i) {
        $b_watchlist = 1;
        $b_browse = 0;
    } elsif ($opt=~/^browse$/i) {
        $b_watchlist = 0;
        $b_browse = 1;
    } else {
        print "Unknown command: '${opt}'\n\n";
        $help = 1;
    }
}
if ($help) {
    print <<HELP;

CRDL - CrunchyRoll DownLoader

Capturing snapshot of "browse all" list, or personal "watchlist"
  Version 0.3, 2026-08-02

Usage:
  'crdl.pl browse' - Current CrunchyRoll shows, alphabetical
  'crdl.pl watchlist' - Current CrunchyRoll personal watchlist
  Options:
    -conf <FILENAME>: Filename of config file (default: '${config_file}')
    -json: Store JSON-responses from the server
    -timestamp <TIMESTAMP>: Use specific timestamp (YYYYMMDDhhmmss format) for output filenames instead of current time

  Provide credentials in crdl.conf, good for just a few minutes:
    1. Log into https://www.crunchyroll.com/, select the desired user
    2. Enter Firefox Inspector (Tools -> Browser Tools -> Web Developer Tools), select 'Network' tab (or equivalent in Chrome)
    3. Reload
    4. Under 'Network' tab in inspector, look for any transfers returning JSON, e.g., the 'me' script (filter by XHR), click on it
    5. Right-Click on it, then select "Copy Value" and "Copy Request Headers"
    6. Paste text into config file and save

HELP
    exit;
}

# Application to download web pages
my $wget = 'wget -t 3 -T 10 --no-check-certificate';

my $wget_options = "--header='Accept-Language: en-US,en;q=0.9' " .
  "--header='Accept: application/json' " .
  "--header='Accept-Encoding: deflate' " .
  "--header='Sec-Fetch-Dest: empty' " .
  "--header='Sec-Fetch-Mode: cors' " .
  "--header='Sec-Fetch-Site: same-origin' " .
  "--header='Pragma: no-cache' " .
  "--header='Cache-Control: no-cache' ";

# Read config file
my $path = dirname(__FILE__); # dirname() requires File::Basename
if (open my $conf, "${path}/${config_file}") {
    while (<$conf>) {
        chomp;
        my $line = $_;
        next if $line=~/^#/;
        if (($line =~/^Authorization:/) || ($line =~/^Cookie:/)) {
            $wget_options .= "--header='${line}' ";
        } elsif ($line =~/^User-Agent: (.*)$/) {
            $wget_options .= "-U '$1' ";
        }
    }
    close $conf;
} else {
    print "Failed to open config file '${path}/${config_file}'! Using built-in defaults, will likely fail.\n\n";
}

# Date and time (YYYYMMDDhhmmss)
sub timestamp
{
    my ($sec, $min, $hour, $mday, $month, $year, $wday, $yday, $isdst) = localtime(time());
    $year += 1900;
    $month++;
    return sprintf("%04u%02u%02u%02u%02u%02u", $year, $month, $mday, $hour, $min, $sec);
}

# Time stamp for file names, representing start time
my $isotime = timestamp();

if ($b_resume) {
    # Read most recent timestamp unless specified on command line
    unless ($timestamp) {
        open TIMESTAMP, "<timestamp.txt" or die "Can't read timestamp.txt file!";
        $timestamp = <TIMESTAMP>;
        chomp $timestamp;
        #print "Timestamp from file: '$timestamp'\n";
        close TIMESTAMP;
    }
    $isotime = $timestamp if ($timestamp);
} else {
    $isotime = $timestamp if ($timestamp);
    # Make most recent timestamp available to follow-up scripts
    open TIMESTAMP, ">timestamp.txt" or die "Can't write timestamp.txt file!";
    print TIMESTAMP "$isotime";
    close TIMESTAMP;
}
#print "Capturing with timestamp '${isotime}'\n";

# File names
my $file_prefix = 'cr_data' . (($b_watchlist) ? '_watchlist' : '_browse') . "_${isotime}";

sub query_list
{
    my ($chunk, $chunk_size, $total) = (0, 0, 0);
    my $series_index = 0;
    my $watchlist_json = '';
    my $series_json = '';
    my $record_count = 0;
    
    my $encoding = '>' . ':encoding(UTF-8)';
    
    my $base_url = 'https://www.crunchyroll.com/content/v2/';

    # Determine currently selected user
    my $watchlist_user = 'https://www.crunchyroll.com/accounts/v1/me';
    my $account_id = '';
    if (open PAGE, "${wget} ${wget_options} '${watchlist_user}' -O - 2>/dev/null |") {
        binmode(PAGE, ":utf8");
        #print "\nQueried: '${watchlist_user}'\n";
        my $response_json = '';
        while (<PAGE>) {
            chomp;
            $response_json .= $_;
        }
        close PAGE;
        #print "JSON: '$response_json'\n\n";
        die "Not logged in!" unless ($response_json =~/^{.*}/);
        my $jref = decode_json(encode_utf8($response_json));
        #for (keys %{$jref}) {
        #    my $value = %{$jref}{$_};
        #    print "key='$_' : value='$value'\n";
        #}
        $account_id = %{$jref}{'account_id'} if (%{$jref}{'account_id'});
    }
    die "Not logged in!" unless ($account_id);
    
    # URLs of CrunchyRoll watchlist tab
    # https://www.crunchyroll.com/watchlist
    my $watchlist_url = 'discover/' . $account_id . '/watchlist';
    my $watchlist_crm = 'cms/objects/';

    # URL of CrunchyRoll browse tab
    # https://www.crunchyroll.com/videos/alphabetical
    my $browse_url = 'discover/browse';
    
    # DataBase file (lines of JSON)
    if ($b_get_json) {
        my $db  = $file_prefix . ".json";
        open DB, $encoding, "${db}" or die "Can't write database '${db}'!";
    }
    
    # Log file, plain text list of shows
    my $log  = $file_prefix . ".txt";
    open LOG, $encoding, "${log}" or die "Can't write logfile '${log}'!";
    
    while (($total == 0) || ($chunk * $chunk_size < $total)) {
        # ?order=desc&n=100&start=100
        # 
        my $query_url = $base_url;
        if ($b_watchlist) {
            $chunk_size = 100;
            $query_url .= $watchlist_url . '?order=desc';
        } elsif ($b_browse) {
            $chunk_size = 50;
            $query_url .= $browse_url . '?sort_by=alphabetical&ratings=true';
        }
        $query_url .= '&start=' . ($chunk * $chunk_size) . '&n=' . ($chunk_size) . 
            '&preferred_audio_language=ja-JP&locale=en-US';
        #        print "${wget} ${wget_options} '${query_url}' -O - 2>/dev/null\n";
        if (open PAGE, "${wget} ${wget_options} '${query_url}' -O - 2>/dev/null |") {
            binmode(PAGE, ":utf8");
            #print "\nQueried: '${query_url}'\n";
            my $response_json = '';
            while (<PAGE>) {
                chomp;
                $response_json .= $_;
            }
            close PAGE;
            #print "JSON: '$response_json'\n\n";
            if ($b_get_json) {
                print DB "${response_json}\n";
            }
            my $jref = decode_json(encode_utf8($response_json));
            #for (keys %{$jref}) {
            #    my $value = %{$jref}{$_};
            #    print "key='$_' : value='$value'\n";
            #}
            my $list = %{$jref}{data};
            $total = %{$jref}{total};
            if ($record_count && ($record_count != $total)) {
                print " - Record count mismatch: was ${record_count}, now ${total}\n";
            }
            $record_count = $total;
            my $data;
            if ($b_watchlist) {
                my $series = '';
                foreach my $record (@{$list}) {
                    my $panel = %{$record}{'panel'};
                    my $ep = %{$panel}{'episode_metadata'};
                    my $id = %{$ep}{'series_id'};
                    $series .= ((($series) ? ',' : '') . $id);
                }
                $query_url = $base_url;
                $query_url .= $watchlist_crm . $series . '?ratings=true';
                $query_url .= '&preferred_audio_language=ja-JP&locale=en-US';
                #        print "${wget} ${wget_options} '${query_url}' -O - 2>/dev/null\n";
                if (open PAGE, "${wget} ${wget_options} '${query_url}' -O - 2>/dev/null |") {
                    binmode(PAGE, ":utf8");
                    #print "\nQueried: '${query_url}'\n";
                    my $series_json = '';
                    while (<PAGE>) {
                        chomp;
                        $series_json .= $_;
                    }
                    close PAGE;
                    #print "JSON: '$series_json'\n\n";
                    if ($b_get_json) {
                        print DB "${series_json}\n";
                    }
                    my $data_jref = decode_json(encode_utf8($series_json));
                    $data = %{$data_jref}{data};
                }
            } elsif ($b_browse) {
            }
            my $data_index = 0;
            foreach my $record (@{$list}) {
                if ($b_watchlist) {
                    #print "Record: '$record'\n";
                    # HASH-ref of keys 'records'(HASH), 'info'(HASH), 'record_count'(SCALAR), 'errors'(HASH)
                    #for (keys %{$record}) {
                    #    my $value = %{$record}{$_};
                    #    print "record key='$_' : value='$value'\n";
                    #}
                    my $b_never_watched = %{$record}{'never_watched'};
                    my $b_fully_watched = %{$record}{'fully_watched'};
                    my $panel = %{$record}{'panel'};
                    #for (keys %{$panel}) {
                    #    my $value = %{$panel}{$_};
                    #    print "panel key='$_' : value='$value'\n";
                    #}
                    my $ep = %{$panel}{'episode_metadata'};
                    #for (keys %{$ep}) {
                    #    my $value = %{$ep}{$_};
                    #    print "episode key='$_' : value='$value'\n";
                    #}
                    my $episode = %{$ep}{'episode'};
                    my $season = %{$ep}{'season_number'};
                    #my $sep = %{$ep}{'sequence_number'};
                    my $series = @{$data}[$data_index];
                    my $title = %{$series}{'title'};
                    my $md = %{$series}{'series_metadata'};
                    #for (keys %{$md}) {
                    #    my $value = %{$md}{$_};
                    #    print "metadata key='$_' : value='$value'\n";
                    #}
                    my $year = %{$md}{'series_launch_year'};
                    my $episodes = %{$md}{'episode_count'};
                    # sed -E 's/^[0-9]*\: //' < cr_watchlist.2026-07-08.txt > scarab_watchlist.2026-07-08.txt
                    #print "${series_index}: ${title} (${year}) - ${episodes} Ep." . 
                    #    ($b_never_watched ? "" : $b_fully_watched ? " - Done" : " - Next: S${season}E${episode}") . 
                    #    "\n";
                    print LOG "${title} (${year}) - ${episodes} Ep." . 
                        ($b_never_watched ? "" : $b_fully_watched ? " - Done" : " - Next: S${season}E${episode}") . 
                        "\n";
                    $data_index++;
                    $series_index++;
                } elsif ($b_browse) {
                    #print "Record: '$record'\n";
                    #for (keys %{$record}) {
                    #    my $value = %{$record}{$_};
                    #    print "record key='$_' : value='$value'\n";
                    #}
                    my $id = %{$record}{'id'};
                    my $title = %{$record}{'title'};
                    my $last_public = %{$record}{'last_public'};
                    my $md = %{$record}{'series_metadata'};
                    #for (keys %{$md}) {
                    #    my $value = %{$md}{$_};
                    #    print "metadata key='$_' : value='$value'\n";
                    #}
                    my $year = %{$md}{'series_launch_year'};
                    my $seasons = %{$md}{'season_count'};
                    my $episodes = %{$md}{'episode_count'};
                    print LOG "${title} (${year}) - ${seasons} Sn. / ${episodes} Ep. - Added: ${last_public}\n";
                    $data_index++;
                    $series_index++;
                }
            }
            print " - Processed ${series_index} of ${record_count}\r";
            $chunk++;
        }
    }
    print LOG "Series: ${series_index} of ${record_count}\n";
    
    close DB if ($b_get_json);
    close LOG;
    
    print "  Captured ${series_index} of ${record_count} series into ${log}\n";
    
    return 0;
}

query_list($timestamp);