#!/usr/bin/perl

eval 'exec /usr/bin/perl  -S $0 ${1+"$@"}'
    if 0; # not running under some shell

#################################################
# Changelog:
# 13/07/2005
# - removed backend lastampa, now using 
#   wfactory.net instead (basically the same)
# - removed backend mytv, site doesn't provide 
#   data anymore. Didn't remove the code since
#   it might come in handy in the future.
# 25/08/2005
# - updated after changes in skytv.it site
# - first test with simple double language messages and docs
# 14/06/2006
# - minor update for changes in skytv.it site (now skylife.it)
# 16/08/2006
# - fixes to skytv
# - skytv now handles categories when using --slow
#################################################
# TODO
# - find channel icons somewhere
# - add more informative errors in xml
#
#################################################

#pod below is handled at install time
=pod

=head1 NOME

tv_grab_it - Recupera informazioni sui programmi TV italiani.

=head1 SINTASSI

tv_grab_it --help

tv_grab_it [--config-file FILE] --configure

tv_grab_it [--config-file FILE] [--output FILE]
           [--days N] [--offset N] [--quiet]
           [--slow] [--verbose] [--errors-in-xml]
           [--backend SITE1[,SITE2[,SITE3]]]
        

=head1 DESCRIZIONE

Crea un file con i programmi TV di vari canali italiani.
Il grabber eE<39> basato sullE<39>analisi del sorgente HTML dei siti, 
quindi potrebbe smettere di funzionare in qualsiasi momento.
I dati vengono presi da piuE<39> fonti per poter ridurre i periodi di
blackout dovuti a cambiamenti nei siti, ma anche per aumentare
il numero di canali disponibili.
Se il grabber non riesce ad ottenere dati dalla prima fonte passeraE<39>
alla seconda, e cosE<39> via. Puoi specificare il tuo ordine usando
lE<39>opzione --backend.

Le fonti configurate al momento sono: (in ordine di utilizzo):

=over

=item B<wfactory> - prende i dati da www.wfactory.net

=item B<skytv>    - prende i dati da www.skytv.it

=back

Per prima cosa esegui B<tv_grab_it --configure> per scegliere quali
canali vuoi scaricare. In seguito lE<39>esecuzione di B<tv_grab_it> 
senza opzioni manderE<39> sullo standard output i dati in formato XML.

B<--configure> Chiede quali canali scaricare e scrive il file di configurazione.

B<--config-file FILE> Imposta il nome del file di configurazione, di 
default eE<39> B<~/.xmltv/tv_grab_it.conf>. Questo file viene scritto
usando B<--configure> e letto durante il grabbing.

B<--gui OPTION> Usa questa opzione per abilitare una interfaccia grafica.
OPTION puE<39> essere 'Tk', oppure lasciato in bianco per la scelta migliore.
Altri valori possibili per OPTION sono: 'Term' per un terminale normale
(default) e 'TermNoProgressBar' per disabilitare lE<39>uso di XMLTV::ProgressBar.

B<--output FILE> scrive in questo file invece che sullo standard output.

B<--days N> prende dati per N giorni. Di default eE<39> 7.

B<--offset N> parte da N giorni in poi. Normalmente parte da oggi.

B<--quiet> non usa i messaggi di avanzamento che normalmente vengono scritti su standard error.

B<--slow> scarica piuE<39> dettagli (trame, attori...). Questo vuol dire scaricare un nuovo file per ogni programma, quindi di default eE<39> disabilitato per risparmiare tempo.

B<--verbose> scrive piuE<39> informazioni su quello che sta facendo il programma, utile per il debugging.

B<--errors-in-xml> scrive gli errori sotto forma di programmi nel file XML, cosiE<39> possono essere visti nel tuo frontend preferito oltre che in STDERR.

B<--backend> imposta la sorgente da utilizzare. Vedi gli esempi.

=head1 ATTENZIONE

Se usi --quiet dovresti usare anche --errors-in-xml, o non avrai nessun
avvertimento per eventuali errori. A differenza delle versioni precedenti,
inoltre, se il grabber non riesce a scaricare nessun dato ritorna un
file XML vuoto (o opzionalmente con solo i warning).

La sorgente mytv backend al momento eE<39> disabilitata (il sito non fornisce piuE<39> dati).

La qualitaE<39> dei dati delle varie sorgenti cambia molto. Per esempio, mytv
era molto semplica, ma completa e usava la minor banda possibile. Skytv ha molti
canali, ma i dati non sono molto buoni a meno che non si usi lE<39>opzione --slow, 
(ed in quel caso serve MOLTO tempo). wfactory eE<39> tutto sommato un buon sito se 
non hai bisogno di tutto i pacchetto sky.

=head1 ESEMPI

=over 

=item tv_grab_it --backend mytv --configure

configura tv_grab_it usando solo la sorgente mytv 

=item tv_grab_it --backend skytv,wfactory --days 1

prende solo un giorno di dati e utilizza un ordine diverso da quello di default (si sarebbe potuto scrivere anche cosiE<39>: --backend skytv --backend wfactory)

=item tv_grab_it --cache --slow --days 3

prende tutti i dati per i prossimi tre giorni ed usa una cache su disco (sempre raccomandabile).

=back

=head1 VEDERE ANCHE

L<xmltv(5)>.

=head1 AUTORE

Davide Chiarini, davide.chiarini@gmail.com

=cut

#default language for warnings set at install time
my $DEF_LANG = 'ita';
######################################################################
# initializations
use warnings;
use strict;

use XMLTV::Version '$Id: tv_grab_it.in,v 1.46 2006/08/16 20:59:24 mnbjhguyt Exp $';
use XMLTV::Capabilities qw/baseline manualconfig cache share/;
use XMLTV::Description 'Italy';
use HTML::Entities;
use HTML::Parser;
use URI::Escape;
use Getopt::Long;
use Date::Manip;
use Memoize;
use XMLTV;
use XMLTV::Memoize;
use XMLTV::Ask;
use XMLTV::Config_file;
use XMLTV::ProgressBar;
use XMLTV::DST;
use XMLTV::Get_nice;
use XMLTV::Mode;

# Todo: perhaps we should internationalize messages and docs?
use XMLTV::Usage <<END
$0: get Italian television listings in XMLTV format
To configure: $0 --configure [--config-file FILE]
To grab listings: $0 [--config-file FILE] [--output FILE] [--days N]
        [--offset N] [--quiet] [--slow] [--verbose] [--backend]
        [--errors-in-xml] 
To list available channels: $0 [--output FILE] [--quiet] --list-channels
To show capabilities: $0 --capabilities
To show version: $0 --version
END
  ;

# Use Log::TraceMessages if installed.
BEGIN {
    eval { require Log::TraceMessages };
    if ($@) {
    *t = sub {};
    *d = sub { '' };
    }
    else {
    *t = \&Log::TraceMessages::t;
    *d = \&Log::TraceMessages::d;
    Log::TraceMessages::check_argv();
    }
}

#max days on the server
my $MAX_DAYS=7;

# default language
my $LANG="it";
my $date_today = UnixDate("today", '%Y-%m-%d');

my @default_backends = ('skytv', 'wfactory');

my %channels; #to store display names

# backend configurations
my %backend_info
  = ( 
     'wfactory' =>
     { domain => 'wfactory.net',
       base_chan => 'http://www.wfactory.net/showcase/programmi.jsp?FRM_SEARCH_DATE='.$date_today.'&FRM_SEARCH_START_TIME=16%3A00&FRM_SEARCH_TYPE=%25%25&FRM_SEARCH_PACK=%25%25&FRM_SEARCH_CHANNEL=%25%25&canale=%2FTelevisione&x=4&y=4',
       base_data => 'http://www.wfactory.net/showcase/search_channel.jsp?',
       rturl     => "http://www.wfactory.net/showcase/programmi.jsp",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub   => \&wfactory_fetch_data,
       channel_list_sub => \&wfactory_get_channels_list,
     },

     'mytv' =>
     { domain => 'my-tv.it',
       base_chan => 'http://www.my-tv.it/palinsesto/guidatv_xml.jsp',
       base_data => 'http://www.my-tv.it/palinsesto/guidatv_xml.jsp',
       rturl => "http://www.my-tv.it/",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&mytv_fetch_data,
       channel_list_sub => \&mytv_get_channels_list,
     },

     'skytv' =>
     { domain => 'skylife.it',
       base_chan => 'http://www.skylife.it/html/guidatv/programmazione/epgCinema.html',
       base_data => 'http://www.skylife.it/html/guidatv/programmazione/epgCinema.html',
       rturl => "http://www.skylife.it/",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&skytv_fetch_data,
       channel_list_sub => \&skytv_get_channels_list,
     },
    );

######################################################################
# Get options, including undocumented --cache option.
my $func_name = 'XMLTV::Get_nice::get_nice_aux';
XMLTV::Memoize::check_argv($func_name) # cache on disk
  or memoize($func_name)               # cache in memory
  or die "cannot memoize $func_name: $!";

my ($opt_days,
    $opt_offset,
    $opt_help,
    $opt_output,
    $opt_slow,
    $opt_verbose,
    $opt_configure,
    $opt_config_file,
    $opt_gui,
    $opt_quiet,
    $opt_share,
    $opt_errors_in_xml,
    @opt_backends,
    $opt_list_channels,
   );

# server only holds 7 days, so if there is an offset days must be
# opt_days-offset or less.

$opt_offset = 0;   # default
$opt_quiet  = 0;   # default
$opt_slow   = 0;   # default
$opt_verbose = 0;  # default

GetOptions('days=i'       => \$opt_days,
       'offset=i'         => \$opt_offset,
       'help'             => \$opt_help,
       'configure'        => \$opt_configure,
       'config-file=s'    => \$opt_config_file,
       'gui:s'            => \$opt_gui,
       'output=s'         => \$opt_output,
       'quiet'            => \$opt_quiet,
       'slow'             => \$opt_slow,
       'verbose'          => \$opt_verbose,
       'share=s'          => \$opt_share,       # undocumented
       'errors-in-xml'    => \$opt_errors_in_xml,
       'backend=s'        => \@opt_backends,
       'list-channels'    => \$opt_list_channels,
      )
  or usage(0);

die ($DEF_LANG eq 'eng' ? 
         "number of days (--days) must not be negative. You gave: $opt_days\n" :
         "il numero di giorni (--days) non puo' essere negativo. Hai specificato: $opt_days\n")
  if (defined $opt_days && $opt_days < 0);

die ($DEF_LANG eq 'eng' ?
        "offset days (--offset) must not be negative. You gave: $opt_offset\n" :
        "l'intervallo di partenza (--offset) non puo' essere negativo. Hai specificato: $opt_offset\n")
  if ($opt_offset < 0);
usage(1) if $opt_help;

if ($opt_quiet) {
    $opt_verbose = 0;
}

$opt_days = $opt_days || $MAX_DAYS;

my $mode = XMLTV::Mode::mode('grab',
			     $opt_list_channels => 'list-channels',
			     $opt_configure => 'configure');

# parse the --backend option
@opt_backends = split(/,/,join(',',@opt_backends)); #we allow both multiple --backend and --backend=name1,name2

my @backends = ();
foreach (@opt_backends) {
    if (defined $backend_info{$_}) {
        push @backends, $_;
    }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "Unknown backend $_!\n" :
                          "Fonte sconosciuta $_!\n"
                         );
    }
}
unless (@backends) {
    @backends = @default_backends;
    if (@opt_backends) {    #we specified backends but we didn't like them, warn the user
        warn ($DEF_LANG eq 'eng' ?
                        "No good backend specified, falling back on defaults\n" : 
                        "Nessuna fonte corretta specificata, uso i default\n"
                        );
    }
}

XMLTV::Ask::init($opt_gui);

# share/ directory for storing channel mapping files.  This next line
# is altered by processing through tv_grab_it.PL.  But we can use
# the current directory instead of share/tv_grab_it for development.
#
# The 'source' file tv_grab_it.in has $SHARE_DIR undef, which means
# use the current directory.  In any case the directory can be
# overridden with the --share option (useful for testing).
#

my $SHARE_DIR='/usr/share/xmltv'; # by grab/it/tv_grab_it.PL
$SHARE_DIR = $opt_share if defined $opt_share;
my $OUR_SHARE_DIR = (defined $SHARE_DIR) ? "$SHARE_DIR/tv_grab_it" : '.';
(my $CHANNEL_NAMES_FILE = "$OUR_SHARE_DIR/channel_ids") =~ tr!/!/!s;

# reads the file channel_ids, which contains the tables to convert 
# between backends' ids and XMLTV ids of channels.
# to support multiple backends i add a ini-style [section] header
# there are two fields: xmltv_id and site_id.

my (%xmltv_chanid, %seen);
my $line_num = 0;
my $backend;
foreach (XMLTV::Config_file::read_lines($CHANNEL_NAMES_FILE, 1)) {
    ++ $line_num;
    next unless defined;
    my $where = "$CHANNEL_NAMES_FILE:$line_num";

    if (/^\[(.*)\]$/) {
        if (defined $backend_info{$1}) {    #esiste la configurazione
            $backend = $1;
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                "Unknown backend $1 in $where\n" :
                                "Fonte sconosciuta $1 in $where\n");
            $backend = undef;
        }
    }
    elsif ($backend) {
        my @fields = split /;/;
        die ($DEF_LANG eq 'eng' ?
                        "$where: wrong number of fields" : 
                        "$where: numero di campi errato")
          if @fields != 2;#3;

        my ($xmltv_id, $site_id) = @fields;

        warn ($DEF_LANG eq 'eng' ?
                          "$where: backend id $site_id for site '$backend' seen already\n" :
                          "$where: fonte con id $site_id per il sito '$backend' gia' visto!\n"
                          )
          if defined $backend_info{$backend}{site_ids}{$xmltv_id};
        $backend_info{$backend}{site_ids}{$xmltv_id}{site_id} = $site_id;
        #$backend_info{$backend}{site_ids}{$xmltv_id}{satellite} = $sat;

        warn ($DEF_LANG eq 'eng' ?
                         "$where: XMLTV_id $xmltv_id for site '$backend' seen already\n" :
                         "$where: XMLTV_id $xmltv_id per il sito '$backend' gia' visto!\n" )
          if $seen{$backend.$xmltv_id}++;
    }
}

# File that stores which channels to download.  Not needed for
# list-channels mode.
#
my $config_file;
unless ($mode eq 'list-channels') {
    $config_file = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_it', $opt_quiet);
}

XMLTV::Config_file::check_no_overwrite($config_file) if $mode eq 'configure';

# Arguments for XMLTV::Writer.
my %w_args;
if (defined $opt_output) {
    die($DEF_LANG eq 'eng' ?
	"cannot give --output with --configure" :
	"non e' possibile specificare --output con --configure")
	if $mode eq 'configure';
    my $fh = new IO::File(">$opt_output");
    die ($DEF_LANG eq 'eng' ?
                 "cannot write to $opt_output: $!" : 
                 "impossibile scrivere su $opt_output") if not defined $fh;
    $w_args{OUTPUT} = $fh;
}
$w_args{encoding} = 'ISO-8859-1';


$line_num = 0;

my $foundchannels;

my $bar = new XMLTV::ProgressBar(($DEF_LANG eq 'eng' ? 'getting list of channels' : 'prendo la lista dei canali'), scalar @backends)
  if not $opt_quiet;
# find list of available channels
foreach $backend (@backends) {
    %{$backend_info{$backend}{channels}} = &{$backend_info{$backend}{channel_list_sub}}($backend_info{$backend}{base_chan});
    $foundchannels+=scalar(keys(%{$backend_info{$backend}{channels}}));

    if (not $opt_quiet) {
        update $bar; 
    }
}
$bar->finish() if (not $opt_quiet);
die ($DEF_LANG eq 'eng' ? "no channels could be found" : "nessun canale trovato") unless ($foundchannels);
warn ($DEF_LANG eq 'eng' ? 
     "VERBOSE: $foundchannels channels found.\n" :
         "VERBOSE: $foundchannels canali trovati.\n") if ($opt_verbose);

######################################################################
# write configuration
if ($mode eq 'configure') {
    open(CONF, ">$config_file") or die ($DEF_LANG eq 'eng' ? 
                                                "cannot write to $config_file: $!" :
                                                "impossibile scrivere su $config_file: $!");

    my %channels;
    foreach $backend (@backends) {
        #faccio un hash con tutti gli id
        foreach (keys %{$backend_info{$backend}{channels}}) {
            $channels{$_} = xmltv_chanid($backend, $_);
        }

        #not used yet
        if ($backend_info{$backend}{needs_login}) {
            say "To get listings on '$backend' you will need a login on the site.\n";
            my $username_wanted = ask_boolean('Do you have a login?', 0);
            if ($username_wanted) {
                $backend_info{$backend}{username} = ask("Username:");
                print CONF "username: $backend:$backend_info{$backend}{username}\n";
            }
        }
    }

    #double reverse to get rid of duplicates
    %channels = reverse %channels;
    %channels = reverse %channels;

    # Ask about each channel.
    my @names = sort keys %channels;
    my @qs = map { ($DEF_LANG eq 'eng' ? "add channel $_?" : "aggiungo il canale $_?") } @names;
    my @want = ask_many_boolean(1, @qs);
    foreach (@names) {
        die if $_ =~ tr/\r\n//;
        my $w = shift @want;
        warn("cannot read input, stopping channel questions"), last
          if not defined $w;
        # No need to print to user - XMLTV::Ask is verbose enough.

        # Print a config line, but comment it out if channel not wanted.
        print CONF '#' if not $w;
        print CONF "channel ".$channels{$_}." # $_\n";
    }

    close CONF or warn ($DEF_LANG eq 'eng' ? 
                                "cannot close $config_file: $!" : 
                                "impossibile chiudere $config_file: $!");
    say(($DEF_LANG eq 'eng' ? "Finished configuration." : "Configurazione terminata."));

    exit();
}

# Not configuring, must be writing some XML.
my $w = new XMLTV::Writer(%w_args);

my $source_info_str = join ",", map {'http://'.$backend_info{$_}{domain}} @backends;
my $source_data_str = join ",", map {$backend_info{$_}{rturl}} @backends;

$w->start({ 'source-info-url'     => $source_info_str ,
        'source-data-url'     => $source_data_str,
        'generator-info-name' => 'XMLTV',
        'generator-info-url'  => 'http://membled.com/work/apps/xmltv/',
        });


my %display_names;
foreach my $back (@backends) {
        foreach (keys %{$backend_info{$back}{site_ids}}) {
                $display_names{$_} = $backend_info{$back}{site_ids}{$_}{site_id};
        }
}

if ($mode eq 'list-channels') {
    # Write all known channels then finish.
    foreach my $xmltv_id (sort keys %display_names) {
        $w->write_channel({
            id => $xmltv_id,
            'display-name' => [ [ $display_names{$xmltv_id} ] ],
#            icon => [{src => get_ch_icon($ch_id)}] #can't find it yet!
        });
    }
    $w->end;
    exit;
}


######################################################################
# read configuration
my @channels;
$line_num = 0;
foreach (XMLTV::Config_file::read_lines($config_file)) {
    ++ $line_num;
    next if not defined;
    if (/^channel:?\s*(.*\S+)\s*$/) {
          push @channels, $1;
    }
    elsif (/^username:?\s+(\S+):(\S+)/){
        if (defined $backend_info{$1}) {    #esiste la configurazione
            $backend_info{$1}{username} = $2;
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                  "Found username for unknown backend $1 in $config_file\n" : 
                                  "Trovato un nome utente per una fonte sconosciuta $1 in $config_file\n");
        }
    }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "$config_file:$line_num: bad line\n" : 
                          "$config_file:$line_num: linea errata\n");
    }
}


######################################################################
# sort out problem in offset options
if ($opt_offset >= $MAX_DAYS) {
    warn ($DEF_LANG eq 'eng' ?
                  "Day offset too big. No program information will be fetched.\n" :
                  "Intervallo specificato troppo grande. Nessun dato verra' scaricato.\n");
    $opt_offset = 0;
    $opt_days = 0;
}
my $days2get;
if (($opt_days+$opt_offset) > $MAX_DAYS) {
    $days2get=$MAX_DAYS-$opt_offset;
    warn ($DEF_LANG eq 'eng' ?
                 "The server only has info for ".($MAX_DAYS-1)." days from today.\n" : 
                 "Il server ha informazioni solo per ".($MAX_DAYS-1)." giorni da oggi.\n");
    if ($days2get > 1) {
        warn ($DEF_LANG eq 'eng' ? 
                          "You'll get listings for only $days2get days.\n" :
                          "Scarico programmi solo per $days2get giorni.\n");
        }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "You'll get listings for only 1 day.\n" : 
                          "Scarico programmi solo per un giorno.\n");
        }
    }
    else {
        $days2get=$opt_days;
    }
t "will get $days2get days from $opt_offset onwards";

## If we are not getting any days of program data we still need to go through
## the loop once to get the channel icons.
#my $last_day;
#if ($days2get == 0) {
#    $last_day = $opt_offset;
#    }
#else {
#    $last_day = $days2get + $opt_offset - 1;
#    }

######################################################################
# grabbing listings

foreach my $xmltv_id (@channels) {
    next if not defined $display_names{$xmltv_id};
    $w->write_channel({
        id => $xmltv_id,
        'display-name' => [ [ $display_names{$xmltv_id} ] ],
#        icon => [{src => get_ch_icon($ch_id)}] #can't find it yet!
        });
}

#make a list of channels and days to grab
my @to_get;
foreach my $day ($opt_offset .. ($days2get + $opt_offset - 1)) {
    foreach my $channel (@channels) {
        push @to_get, [$channel, $day];
    }
}

$bar = new XMLTV::ProgressBar(($DEF_LANG eq 'eng' ? 'getting listings' : 'scarico programmi'), scalar @to_get)
  if not $opt_quiet;

## If we aren't getting any days of program data then clear out the list
## that was created to fetch icons.
#if ($days2get == 0) {@to_get = ();}

foreach (@to_get) {
    my $day     = $_->[1];
    my $channel = $_->[0];

    #this is where i would handle cookies and logins if needed
    warn ($DEF_LANG eq 'eng' ? 
                  "VERBOSE: Grabbing channel $channel, day $day\n" :
                  "VERBOSE: Prendo dati per il canale $channel, giorno $day\n") if ($opt_verbose);
 
    my $error;
    foreach $backend (@backends) {
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Trying with $backend\n" :
                          "VERBOSE: Provo con $backend\n") if ($opt_verbose);

        my @dati; $error = 0;
        ($error, @dati) = &{$backend_info{$backend}{fetch_data_sub}}($channel, $day);

        #TODO different kinds of errors?
        if ($error) {
            warn ($DEF_LANG eq 'eng' ?
                                  "VERBOSE: Error fetching channel $channel day $day with backend $backend\n" :
                                  "VERBOSE: Errore nello scaricare i dati per $channel, giorno $day con $backend\n") if ($opt_verbose);
        } 
        else {
            $w->write_programme($_) foreach @dati;
            last;
        }
    }

    #nessuno ci e' riuscito
    if ($error) {
        #this is an easier way to know about errors if all of our scripts are automated
        if ($opt_errors_in_xml) {
            $w->write_programme(
                {
                    title   => [[($DEF_LANG eq 'eng' ? 'ERROR FETCHING DATA' : 'ERRORE DI SCARICAMENTO DATI'), $LANG]],
                    start   => xmltv_date('00:01', $day),
                    stop    => xmltv_date('23:59', $day),
                    channel => $channel,
                    desc    => [[($DEF_LANG eq 'eng' ?
                                                          "XMLTV couldn't grab data for $channel, day $day. Sorry about that." :
                                                          "XMLTV non e' riuscito a scaricare i dati per $channel, giorno $day. Spiacente."), $LANG]],
                }
            );
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                  "I couldn't fetch data for channel $channel, day $day from any backend!!\n" :
                                  "Non sono riuscito a scaricare i dati per $channel, giorno $day da nessuna fonte!!\n") if (not $opt_quiet);
        }
    }

    update $bar if not $opt_quiet;
}
$w->end;
$bar->finish() if not $opt_quiet;

#####################
# general functions #
#####################

####################################################
# xmltv_chanid
# to handle channels that are not yet in the channel_ids file
sub xmltv_chanid {
    my ($backend, $channel_id) = @_;
    my %chan_ids;

    #reverse id hash
    foreach my $xmltv_id (keys %{$backend_info{$backend}{site_ids}}) {
        my $site_id = $backend_info{$backend}{site_ids}{$xmltv_id}{site_id};
        $chan_ids{$site_id} = $xmltv_id;
        next if (not defined $site_id);
        }

    if (defined $chan_ids{$channel_id}) {
        return $chan_ids{$channel_id};
        }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "***Channel |$channel_id| for '$backend' is not in channel_ids, should be updated.\n" : 
                          "***Il canale |$channel_id| su '$backend' non e' in channel_ids, andrebbe aggiornato.\n"
                          ) unless $opt_quiet;
        $channel_id=~ s/\W//gs;

        #make up an id
        my $id = lc($channel_id).".".$backend_info{$backend}{domain};

    ##update backend info
        #$backend_info{$backend}{site_ids}{$id}{site_id} = $channel_id;
        return $id;
    }


}

##########################################################
# tidy
# decodes entities and removes some illegal chars
sub tidy($) {
    for (my $tmp=shift) {
    s/[\000-\037]//g;   # remove control characters
    s/[\222]/\'/g;      # messed up char
    s/[\224]/\"/g;      # end quote
    s/[\205]/\.\.\./g;  # ... must be something messed up in my regexps?
    s/[\223]/\"/g;      #start quote
    s/[\221]/\'/g;

    if (s/[\200-\237]//g) {
        if ($opt_verbose){
            warn ($DEF_LANG eq 'eng' ?
                                  "VERBOSE: removing illegal char: |\\".ord($&)."|\n" : 
                                  "VERBOSE: tolgo carattere illegale: |\\".ord($&)."|\n");
         }
    }

    # Remove leading white space
    s/^\s*//;
    # Remove trailing white space
    s/\s*$//;
    return decode_entities($_);
    }
}


######################
# my-tv.it functions #
######################

####################################################
# mytv_get_channels_list
# returns hash of channel details. which, basically is
# just 'this channel is in the satellite page or the 
# terrestrial page'
sub mytv_get_channels_list {
    my %chan_hash;
    my $base = shift;

    for my $t (0, 1) {
        my $content;

        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: Getting channel list from $base?d=0&t=$t\n" :
                          "VERBOSE: Prendo la lista dei canali da $base?d=0&t=$t\n"
                          )if ($opt_verbose);

                eval { $content = get_nice("$base?d=0&t=".$t); };
                if ($@) {   #get_nice has died
                        warn ($DEF_LANG eq 'eng' ?
                                                          "VERBOSE: Cannot get mytv's channel list ($base?d=0&t=$t). Site down?\n" :
                                                          "VERBOSE: Non sono riuscito a prendere la lista dei canali di mytv ($base?d=0&t=$t). Il sito non funziona?\n"
                                                          );

                        return ();
                } 

        $content=~/<data>(.*)<\/data>/s;
        $content = $1;

        #split the lines
        my @lines = split /\n/, $content; #shift @lines;
        foreach (@lines) {
            if (/ can="(.*?)"/) {
                $chan_hash{$1}="$t";
    
                #update backend info, in case this is a new channel not in channel_ids
                my $xmltv_id = xmltv_chanid('mytv', $1);
                $backend_info{mytv}{site_ids}{$xmltv_id}{site_id}   = $1;
                $backend_info{mytv}{site_ids}{$xmltv_id}{satellite} = $t;
            }
        }
    }

    return %chan_hash;
}

####################################################
# mytv_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub mytv_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    # build url to grab
    # my-tv is a very simple site: it has two pages per day
    # (one for satellite, one for terrestrial channels)
    # so we are basically fetching the same page over and over
    # this is not a problem as XMLTV caches the fetches.

    # the channels hash holds 1/0 if the channel is sat/ter
    my $site_id = $backend_info{mytv}{site_ids}{$xmltv_id}{site_id};

    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    my $sat = $backend_info{mytv}{channels}{$site_id};
    $sat = 1 if (not defined $sat); #if we don't know we try to guess!;

    my $url = $backend_info{mytv}{base_data}."?d=".$offset."&t=".$sat;
    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend mytv\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte mytv\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    $content=~/<data>(.*)<\/data>/s;
    $content = $1;

    #split the lines
    my @lines = split /\n/, $content; shift @lines;
    foreach my $line (@lines) {
        my %programme = ();
        my ($title, $time_start, $chan, $time_end, $year, $country, $category, $category2, $sub_title);
        $line=~/^<row (.*) \/>$/s;

        my @d = split /\" /, $1;
        foreach my $c (@d) {
            my ($key, $val) = split /=\"/, $c;
            next if (not defined $val);
            if ($val=~/(.*)\"$/) {$val = $1;}

            for ($key) {
              /tit/ && do {$title = $val; last};
              /anno/ && do {$year = $val if ($val != 0); last};
              /gen/ && do {$category = $val; last};
              /orain/ && do {$time_start = $val; last};
              /orafi/ && do {$time_end = $val; last};             
              /can/ && do {$chan = $val; last};           
              /ep/ && do {$sub_title = $val; last};           
              /cat/ && do {$category2 = $val; last};              
              /naz/ && do {last}; #same as cat
              /idprg/ && do { last};              
              /idev/ && do { last}; 
              /data/ && do { last};

              warn ($DEF_LANG eq 'eng' ?
                                    "unhandled attribute $key\n" :
                                    "attributo sconosciuto $key\n");
            }
        }

        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn 'no title found, skipping programme';
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn "no start time for title $title, skipping programme";
            next;
        }
            $programme{start}=xmltv_date($time_start, $offset);
        if (not defined $chan) {
            warn "no channel for programme $title at $time_start, skipping programme";
            next;
        }

        my $past_midnight = 0;
        $time_end =~/.......... (..).(..)/; my $time_end2 = "$1.$2";
        $time_start =~/.......... (..).(..)/; my $time_start2 = "$1.$2";
        $past_midnight = 1 if ($time_end2 < $time_start2); #they can work as decimals, 0.32 < 23.44

        next if ($chan ne $backend_info{mytv}{site_ids}{$xmltv_id}{site_id});

        $programme{channel}=$xmltv_id;#"$chan";
        $programme{category}=[[tidy($category), $LANG ]] if defined $category;
        $programme{'sub-title'}=[[$sub_title, $LANG] ] if (defined $sub_title);
        push (@{$programme{category}}, [tidy($category2), $LANG ]) if defined $category2;
        $programme{country} = [[$country, $LANG]] if (defined $country);
        $programme{date} = $year if (defined $year);
        $programme{stop}=xmltv_date($time_end, $offset + $past_midnight) if (defined $time_end);

        #put info in array, if it is wanted
        push @programmes, {%programme};
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

####################################################
# xmltv_date
# this returns a date formatted like 20021229121300 CET
# first argument is time (like '14:20')
# second is date offset from today
sub xmltv_date {
    my ($time, $offset) = @_;

    $time =~/([0-9]+?):([0-9]+).*/ or die ($DEF_LANG eq 'eng' ? "bad time $time" : "strano orario $time");
    my $hour=$1; my $min=$2;

    my $data = &DateCalc("today","+ ".$offset." days");
    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo data') if not defined $data;
    return utc_offset(UnixDate($data, '%Y%m%d').$hour.$min.'00', '+0100');
}


#########################
# wfactory.it functions #
#########################

####################################################
# wfactory_get_channels_list
sub wfactory_get_channels_list {
    my %chan_hash;
    my $base = shift;
    
    my $content;
    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: Getting channel list from $base\n" :
                  "VERBOSE: Scarico la lista dei canali da $base\n") if ($opt_verbose);

        eval { $content = get_nice($base); };
        if ($@) {   #get_nice has died
                warn ($DEF_LANG eq 'eng' ? 
                                          "VERBOSE: Cannot get wfactory's channel list ($base). Site down?\n" :
                                          "VERBOSE: Non sono riuscito a prendere la lista dei canali di wfactory ($base). Il sito non funziona?\n"
                                         );
                return ();
        } 


    my @lines = split /</, $content;

    foreach my $l (@lines) {
        if ($l=~/\Qa href='javascript:ShowChannel(\E(.*?),\"(.*?)\"\)/) {
            $chan_hash{$2}=$1;

            #update backend info, in case this is a new channel not in channel_ids
            #my $xmltv_id = xmltv_chanid('wfactory', $2);
            #$backend_info{wfactory}{site_ids}{$xmltv_id}{site_id} = $1;
        };
    }

    return %chan_hash;
}

####################################################
# wfactory_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub wfactory_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{wfactory}{site_ids}{$xmltv_id}{site_id};

        if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
#   my %chan = reverse %{$backend_info{wfactory}{channels}};
    my %chan = %{$backend_info{wfactory}{channels}};
    my $channel_name  = $backend_info{wfactory}{site_ids}{$xmltv_id}{site_id};
    my $channel_num = $chan{$channel_name}; 
       $channel_name=~s/ /%20/g;

        if (not defined $channel_num) {
                # if we get here it means that the site should have the channel (it's in channel_ids)
                # but for some reason we are missing it's site id (probably the site is down)
                # we return an error so that another backend will by used, if possible
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: \tThis site appears to be down!\n" :
                          "VERBOSE: \tQuesto sito non sembra funzionare!!\n") if ($opt_verbose);
        return (1, ());
        }

    my $date_grab = &DateCalc("today","+ ".$offset." days");

#    use Data::Dump;
#    warn Data::Dump::dump(\%chan)."\n";
#    warn "num $channel_num\nname $channel_name\n";
#    exit;

    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo di data') if not defined $date_grab;
    $date_grab = UnixDate($date_grab, '%Y-%m-%d');

    my $url = $backend_info{wfactory}{base_data}.'FRM_CHAN_CHAN='.$channel_num.'&FRM_CHAN_DATE='.$date_grab.'&FRM_CHAN_NAME='.$channel_name;
    warn ($DEF_LANG eq 'eng' ? 
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend wfactory\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte wfactory\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    $content=~/<td>..\Q<table cellspacing=0 cellpadding=0 border=0 width='100%'>\E(.*?)\Q<\/table>\E/s;
    $content = $1;

    #split and parse
    my @lines = split /\n/, $content;# shift @lines;
    while ( my ($l1, $l2) = splice(@lines, 1, 2)) {
        next unless ($l2);

        my %programme = ();
        my ($title, $time_start, $time_end, $id);

        $l1 =~ /bottom\'>(.*)<\/td>\E/;
        ($time_start, $time_end) = split /-/, $1;

        $l2 =~ /ShowPopUp\((.*)\)'>(.*?)</;
        ($id, $title) = ($1, $2);

        my $past_midnight = 0;
        $past_midnight = 1 if ($time_end < $time_start); #they can work as decimals, 0.32 < 23.44

        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn ($DEF_LANG eq 'eng' ? 'no title found, skipping programme' : 'titolo non trovato, salto');
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn ($DEF_LANG eq 'eng' ? "no start time for title $title, skipping programme" : "nessun orario di inizio per $title, salto");
            next;
        }
            $time_start=~s/\./:/;
            $programme{start}=xmltv_date($time_start, $offset);
        if (not defined $xmltv_id) {
            warn ($DEF_LANG eq 'eng' ? "no channel for programme $title at $time_start, skipping programme" : "canale non trovato per $title alle $time_start, salto");
            next;
        }

        $programme{channel}=$xmltv_id;
        $time_end=~s/\./:/;

        $programme{stop}=xmltv_date($time_end, $offset + $past_midnight);

        if ($opt_slow) {
            wfactory_fetch_data_slow($id, \%programme);
        }

        #put info in array
        push @programmes, {%programme};
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

sub wfactory_fetch_data_slow {
    my ($id, $programme) = @_;

    my $content;
    my $url = 'http://www.wfactory.net/showcase/showtread.jsp?ID='.$id;

    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: getting --slow data from $url" :
                  "VERBOSE: scarico dati --slow da $url") if ($opt_verbose); 
    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: there was some error!" :
                          "VERBOSE: c'e' stato un errore!") if ($opt_verbose);
        return 0;
    }

    if ($content=~/\Q<div name='descript'\E.*?>(.*?)</) {
        my $desc = $1;
        $desc=~s/^\s+//; $desc=~s/\s+$//;
        $programme->{desc}=[[tidy($desc), $LANG] ] if ($desc ne '');
    }

    if ($content=~/\Q<img src='\E(.*?)=yes\'/) {
        #TODO check &amp;
        $programme->{icon}= [{src => 'http://www.wfactory.net/showcase/'.$1.'=yes'}];
    }

    if ($content=~/\Q<!---->\E(.*?)<\/td>/s) {
        my $linetemp=$1;
        $linetemp=~s/^\s+//; $linetemp=~s/\s+$//;
        $linetemp=~/(.*)\Q&nbsp;\E(.*)\Q&nbsp;\E(.*)/;
        my ($category, $year, $country) = ($1, $2, $3); 
        
        $programme->{date} = $year if (defined $year and $year ne '');
        $programme->{country} = [[$country, $LANG]] if (defined $country and $country ne '');
	
	#it looks like category is used only in movies
	if (defined $category and $category ne '') {
	    $programme->{category}=[['Film', $LANG ]];
	    push (@{$programme->{category}}, [tidy($category), $LANG ]);
	}
	
#       warn "|$category|$year|$country\n";
    }

    my @lines = split /\n/, $content;
    foreach my $l (@lines) {
        $l = tidy($l);

        if ($l=~/^<b>(.*?)<\/b>(.*)/mgi) {
            my ($cat, $val) = ($1, $2);
            $val=~s/<br>$//;
            for ($cat){
                /Regia:/ && do {
                    my @directors = split /, /, $val;
                    foreach $a (@directors) {
                        $a=~s/^\s+//; $a=~s/\s+$//;
                        push @{$programme->{credits}->{director}}, $a;
                        }
                    last;
                    };

                /Con:|Con la voce di:/ && do {
                    my @cast = split /,/, $val;
                    foreach (@cast) {
                        s/^\s+//; s/\s+$//;
                        (push @{$programme->{credits}->{actor}}, $_);
                        }
                    last;
                    };

                /Condotto da:|A cura di:/ && do {
                    my @cast = split /,/, $val;
                    foreach (@cast) {
                        s/^\s+//; s/\s+$//;
                        (push @{$programme->{credits}->{presenter}}, $_);
                        }
                    last;
                    };

                /Episodio:/ && do {
                    $val=~s/^\s+//; $val=~s/\s+$//;
                    $programme->{'sub-title'}=[[$val, $LANG] ];
                    last;
                    };

                warn ($DEF_LANG eq 'eng' ? 
                                          "Don't know what |$cat|$val| is\n" : 
                                          "Non conosco |$cat|$val|\n");
            }
        }
    }
}

####################
# skytv.it functions #
####################

####################################################
# skytv_get_channels_list
sub skytv_get_channels_list {
    my %chan_hash;

    my @urls = (
            #'http://www.skytv.it/GuidaTv/LaProgrammazione/_Cinema.htm',
	    #'http://www.skytv.it/GuidaTv/LaProgrammazione/_Sport.htm',
	    #'http://www.skytv.it/GuidaTv/LaProgrammazione/_News.htm',
	    #'http://www.skytv.it/GuidaTv/LaProgrammazione/_Mondo_e_Tendenze.htm',
	    #'http://www.skytv.it/GuidaTv/LaProgrammazione/_Ragazzi_e_Musica.htm',
	    #'http://www.skytv.it/GuidaTv/LaProgrammazione/_Intrattenimento.htm',
	    'http://www.skytv.it/GuidaTV/LaProgrammazione/tvGuidePopUp?temaTVGuide=1', #was: cinema
	    'http://www.skytv.it/GuidaTV/LaProgrammazione/tvGuidePopUp?temaTVGuide=2', #was: MondiCulture
	    'http://www.skytv.it/GuidaTV/LaProgrammazione/tvGuidePopUp?temaTVGuide=3', #was: intrattenimento
	    'http://www.skytv.it/GuidaTV/LaProgrammazione/tvGuidePopUp?temaTVGuide=4', #was: ragazzimusica
	    'http://www.skytv.it/GuidaTV/LaProgrammazione/tvGuidePopUp?temaTVGuide=5', #was: news
	    'http://www.skytv.it/GuidaTV/LaProgrammazione/tvGuidePopUp?temaTVGuide=6', #was: sport
        );

    foreach my $url (@urls) {
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: Getting channel list from $url\n" :
                          "VERBOSE: Scarico la lista dei canali da $url\n") if ($opt_verbose);
 
        my $content;

                eval { $content = get_nice($url); };
		#warn $content;
		#exit;
                if ($@) {   #get_nice has died
                        warn ($DEF_LANG eq 'eng' ? 
                                                          "VERBOSE: Cannot get skytv's channel list ($url). Site down?\n" : 
                                                          "VERBOSE: Non sono riuscito a prendere la lista dei canali di skytv ($url). Il sito non funziona?\n");
                        return ();
                } 


        my @lines = split /\n/, $content;
        
        my $rowcount = 1;
        foreach my $l (@lines) {
            if ($l=~/\Q<span class=testocanaleTvGuide>\E(.*?)</) {
                $rowcount++;

                my $chan = $1; $chan=~s/^\s+//; $chan=~s/\s+$//;
                next if ($chan=~/^MC/); #we skip them as they have no real pg

                $chan_hash{$chan} = "$url;$rowcount";
    
                #update backend info, in case this is a new channel not in channel_ids
                my $xmltv_id = xmltv_chanid('skytv', $chan);
                $backend_info{skytv}{site_ids}{$xmltv_id}{site_id} = $chan;
            }
        }
    }

    return %chan_hash;
}


sub skytv_fetch_data {
    #this is the worst site design i've ever seen since the <blink> tag days.
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{skytv}{site_ids}{$xmltv_id}{site_id};
    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
    my ($url, $row) = split /;/, $backend_info{skytv}{channels}{$site_id};
    my $date_prev = skytv_urldata($offset-1).'%7C'.($offset); #yesterday
    my $date = skytv_urldata($offset).'%7C'.($offset+1);
    my $num;
    for ($url) {
        #/Mondo_e_Tendenze.htm/ && do{ $num = 2; last;};
	#/Ragazzi_e_Musica.htm/ && do{ $num = 4; last;};
	#/Intrattenimento.htm/ && do{ $num = 3; last;};
	#/News.htm/ && do{ $num = 5; last;};
	#/Sport.htm/ && do{ $num = 6; last;};
	#/Cinema.htm/ && do{ $num = 1; last;};
        /temaTVGuide=(.)$/ && do{ $num = $1; last;};
    }

    # date to grab
    my $grabdate      = UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y%m%d');
    my $grabdate_prev = UnixDate(&DateCalc("today","+ ".($offset - 1)." days"), '%Y%m%d');

    my @urls;
    #for midnight to 06 am we need to get yesterday night's stuff.
    if ($offset > 0) {
        #date_prev
        push @urls, [$url.'?giorno='.$date_prev.'&orario=3&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=20%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date_prev, '22:00', -1];
        push @urls, [$url.'?giorno='.$date_prev.'&orario=3&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=22%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date_prev, '00:00', 0];
        push @urls, [$url.'?giorno='.$date_prev.'&orario=1&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=00%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date_prev, '02:00', 0];
        push @urls, [$url.'?giorno='.$date_prev.'&orario=1&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=02%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date_prev, '04:00', 0];
    }
    push @urls, [$url.'?giorno='.$date.'&orario=1&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=04%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '06:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=1&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=06%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '08:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=1&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=08%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '10:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=1&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=10%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '12:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=2&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=12%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '14:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=2&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=14%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '16:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=2&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=16%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '18:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=3&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=18%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '20:00', 0];
    push @urls, [$url.'?giorno='.$date.'&orario=3&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=20%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '22:00', 0];
    #this is technically tomorrow, but i need to know at what time the last prog ends
    push @urls, [$url.'?giorno='.$date.'&orario=3&tema=1&temaTVGuide='.$num.'&TypeSearch=2&TimeSearch=2&strStartTimeHidden=22%3A00%3A00&temaTimeNavigation='.$num.'&giornoTimeNavigation='.$date, '24:00', 0];

    my @prog_to_check = ();
    foreach (@urls) {
        my $url2 = $_->[0];
        my $starttime = $_->[1];
        my $realday = $_->[2];

        if ($realday == -1) {
            $starttime = ParseDate("$grabdate_prev $starttime");
        }
        else {
            $starttime = ParseDate("$grabdate $starttime");
        }

        die 'date calculation failed' if (! $starttime);

        warn "VERBOSE: fetching $url2\n"  if ($opt_verbose);

        eval { $content=get_nice($url2) };
        if ($@) {   #get_nice has died
                warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend skytv\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte skytv\n") if ($opt_verbose);

            # Indicate to the caller that we had problems
            return (1, ());
        } 

        warn "VERBOSE: parsing...\n" if ($opt_verbose);

        #split and parse the lines
        my @lines = split /\n/, $content;
        my $rowcount = 1;
        foreach my $line (@lines) {
            if ($line=~/cellasxEvento/) {
                $rowcount++;
                if ($rowcount == $row){
                    $line=~s/<td class/\n<td class/gm;
                    my @lines2 = split /\n/, $line;
                    foreach my $line2 (@lines2) {
                        if ($line2=~/cellaCenterEvento/) {
                            if ($line2=~/.*colspan=\"(\d+?)\".*OpenEvent\((\d+?),.*title=\"(.*?)\"/){
                                my ($title, $length, $id) = ($3, $1, $2);
                                $title=~s/^\s+//; $title=~s/\s+$//;
                                my $start = $starttime;
                                $starttime = &DateCalc($starttime, "+ ".$length." minutes") || die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo data');
                                my $stop = $starttime;
                                push @prog_to_check, [$title, $length, $id, $start, $stop];
                            }
                            elsif ($line2=~/.*colspan=\"(\d+?)\"/){
                                if ($line2!~/prev/ and $line2!~/next/) {
                                    $starttime = &DateCalc($starttime, "+ ".$1." minutes") || die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo data');
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    my @programmes = ();
    my ($oldtitle, $oldlength, $oldid, $oldstart, $oldstop) = ('', '', 0, '', '');

    #we join the first prog of the page with the last of the previous page if they are the same
    my %programme = ();

    my $parse_date = &DateCalc("today 00:00","+ ".$offset." days");
    my $next_day   = &DateCalc("today 00:00","+ ".($offset + 1)." days");

    foreach (@prog_to_check) {
        my ($title, $length, $id, $start, $stop) = @$_;
	#warn "($title, $length, $id, $start, $stop)\n";
        if ($id != $oldid) {# and $title ne $oldtitle) {
            #write data
            if ($oldid != 0) {
                #if (substr($oldstart, 6, 2) < $num_date){
                if (Date_Cmp(ParseDate($oldstart), ParseDate($parse_date)) <= 0) {

                    #i get rid of yesterday's programmes, even if they end today (i.e. across midnight shows)
                    ($oldtitle, $oldlength, $oldid, $oldstart, $oldstop) = @$_;
                    next;
                }

                $programme{title} = [[tidy($oldtitle), $LANG] ];
                $programme{start} = utc_offset(UnixDate($oldstart, '%Y%m%d%H%M').'00', '+0100');
                $programme{stop}  = utc_offset(UnixDate($oldstop, '%Y%m%d%H%M').'00', '+0100');
                $programme{channel} = $xmltv_id;
                skytv_fetch_data_slow($oldid, \%programme) if ($opt_slow);
                push @programmes, {%programme};
            }
            %programme = ();
            ($oldtitle, $oldlength, $oldid, $oldstart, $oldstop) = @$_;
            
            #we don't care about tomorrow's data
             last if (Date_Cmp(ParseDate($oldstart), ParseDate($next_day)) > 0); #abbiamo cambiato giorno
        }
        else {
            $oldstop = $_->[4];
        }
    }

    #the programme might end exactly at midnight
    if (Date_Cmp(ParseDate($oldstart), ParseDate($parse_date)) == 0){
        $programme{title} = [[tidy($oldtitle), $LANG] ];
        $programme{start} = utc_offset(UnixDate($oldstart, '%Y%m%d%H%M').'00', '+0100');
        $programme{stop}  = utc_offset(UnixDate($oldstop, '%Y%m%d%H%M').'00', '+0100');
        $programme{channel} = $xmltv_id;
        skytv_fetch_data_slow($oldid, \%programme) if ($opt_slow);
        push @programmes, {%programme} if ($oldid != 0);
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

sub skytv_urldata {
    my $time_offset = shift;
    my $data=&DateCalc("today","+ ".$time_offset." days");
    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo data') if not defined $data;

    my $str=UnixDate($data, '%a+%e');

    #traduciamo in italiano
    $str=~s/Sun/Domenica/;
    $str=~s/Mon/Luned%C3%AC/;
    $str=~s/Tue/Marted%C3%AC/;
    $str=~s/Wed/Mercoled%C3%AC/;
    $str=~s/Thu/Gioved%C3%AC/;
    $str=~s/Fri/Venerd%C3%AC/;
    $str=~s/Sat/Sabato/;

        $str=~s/ //;

    return $str;
}

sub skytv_fetch_data_slow {
    my ($id, $programme) = @_;

    my $content;
    my $url = 'http://www.skytv.it/GuidaTV/LaProgrammazione/Event.htm?idevento='.$id;

    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: getting --slow data from $url" :
                  "VERBOSE: prendo dati --slow da $url") if ($opt_verbose); 
    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: there was some error!" :
                          "VERBOSE: c'e' stato qualche errore!") if ($opt_verbose);
        return 0;
    }

    #get rid of what we don't need
    $content=~/\Q<!-- CONTENUTO CENTRALE -->\E(.*)\Q<!-- FINE CONTENUTO CENTRALE -->\E/s;
    $content = $1;
#warn $content."\n";
#exit;
    if ($content=~/\Q<tr valign=top><td><span class="event_titolo" style="font-weight:bold">\E(.*?)\Q<\/span>&nbsp;<span class="event_titolo">\E(.*?)\Q<\/span>&nbsp;<\/span><\/td><\/tr>\E/) {
        push (@{$programme->{category}}, [tidy($1), $LANG ]) if ($1 ne '');
	push (@{$programme->{category}}, [tidy($2), $LANG ]) if ($2 ne '');
    }
   
    if ($content=~/\Q<tr><td align=right height=30% class="Verdana10_000000">\E(\d+?) min.</) {
        $programme->{length}= $1*60 if ($1 ne '');
    }
    
   
    if ($content=~/\Q<span class="event_sinossi">\E(.*?)</) {
	my $desc = $1;
	my ($cast, $country, $director, $year, $length);

        if ($desc=~/(.*) - (.*)/) {
            $programme->{'sub-title'}=[[$1, $LANG] ] if ($1 ne '');
            $desc = $2 if ($2 ne '');
        }
        if ($desc=~/^Regia di (.*?), con (.*?); (.*?) (\d+?)\. (.*)/) {
            $director = $1;
            $cast = $2;
            $country = $3;
            $year = $4;
            $desc = $5 || '';
        }

        if ($desc=~/^Regia di (.*?), con (.*?); (.*?) (\d+?) \((\d+) min\)\. (.*)/) {
            $director = $1;
            $cast = $2;
            $country = $3;
            $year = $4;
            my $length = $5;
            $desc = $6 || '';
        }

        if ($desc=~/^Con (.*?); (.*?)\.(.*)/) {
            $cast = $1;
            $country = $2;
            $desc = $3 || '';
	}

	#tricky one
        if ($desc=~/^con (.*?)\. (.*)/) {
	   $desc = $2;
	   $cast = $1;
	   
	   if ($cast=~/(.*?); (.*)/) {
    		$cast = $1;
		$country = $2;
	    }
        }
	
	if ($cast) {
	   my @cast = split /,/, $cast;
           foreach (@cast) {
                s/^\s+//; s/\s+$//;
                (push @{$programme->{credits}->{actor}}, $_);
            }
	}
	
	
    $content=~s/[\n|\r]+//gm;	
    
    if ($content=~/\Qtd><span class="event_titolo" style="FONT-WEIGHT: bold">\E(.*?)<\Q\/span>&nbsp;<span class="event_titolo">\E(.*?)</) {
	my ($category, $category2) = ($1, $2);
	
	if ($category=~/Informazione|Intrattenimento|Ragazzi e Musica|Mondo e Tendenze/) {
	    $category = $category2;
	    undef $category2;
	}
    
	$programme->{category}=[[tidy($category), $LANG ]] if defined $category;
        push (@{$programme->{category}}, [tidy($category2), $LANG ]) if defined $category2;
    }

	

        $programme->{length}= $length*60 if ($length);
	$programme->{date}= $year if ($year);
        push @{$programme->{credits}->{director}}, $director if ($director);
        push (@{$programme->{country}}, [$country, $LANG]) if ($country);
        $programme->{desc}=[[tidy($desc), $LANG ]] if ($desc ne '');
    }

}
