#!/usr/bin/perl

eval 'exec /usr/bin/perl  -S $0 ${1+"$@"}'
    if 0; # not running under some shell

#################################################
# Changelog:
# 13/07/2005
# - removed backend lastampa, now using 
#   wfactory.net instead (basically the same)
# - removed backend mytv, site doesn't provide 
#   data anymore. Didn't remove the code since
#   it might come in handy in the future.
# 25/08/2005
# - updated after changes in skytv.it site
# - first test with simple double language messages and docs
# 14/06/2006
# - minor update for changes in skytv.it site (now skylife.it)
# 16/08/2006
# - fixes to skytv
# - skytv now handles categories when using --slow
# 11/01/2007
# - added backend boingtv
# - new option --cache-slow
# 13/02/2007
# - added backend skylife (soon to replace skytv)
# 27/05/2007
# - fixed boingtv.it after site change (thanks Paolo Asioli)
# 02/07/2007
# - fixed skylife after site change (thanks Marco Coli)
# 20/09/2007
# fixes for warnings when in quiet mode
# 06/11/2007
# added backend mtv.it, as skylife strangely doesn't carry it, and wfactory is a stuttering site.
# 08/12/2007
# skylife.it has moved to guidatv.sky.it
# code cleanup
# 15/01/2007
# major optimizations in skylife.it! (thanks Massimo Savazzi)
#################################################
# TODO
# - add more informative errors in xml
#################################################

#pod below is handled at install time
=pod

=head1 NOME

tv_grab_it - Recupera informazioni sui programmi TV italiani.

=head1 SINTASSI

tv_grab_it --help

tv_grab_it [--config-file FILE] --configure

tv_grab_it [--config-file FILE] [--output FILE]
           [--days N] [--offset N] [--quiet]
           [--slow] [--verbose] [--errors-in-xml]
           [--backend SITE1[,SITE2[,SITE3]]]
	   [--cache-slow]
        

=head1 DESCRIZIONE

Crea un file con i programmi TV di vari canali italiani.
Il grabber eE<39> basato sullE<39>analisi del sorgente HTML dei siti, 
quindi potrebbe smettere di funzionare in qualsiasi momento.
I dati vengono presi da piuE<39> fonti per poter ridurre i periodi di
blackout dovuti a cambiamenti nei siti, ma anche per aumentare
il numero di canali disponibili.
Se il grabber non riesce ad ottenere dati dalla prima fonte passeraE<39>
alla seconda, e cosE<39> via. Puoi specificare il tuo ordine usando
lE<39>opzione --backend.

Le fonti configurate al momento sono: (in ordine di utilizzo):

=over

=item B<skylife>  - prende i dati da www.skylife.it

=item B<mtvit>    - prende i dati da www.mtv.it

=item B<wfactory> - prende i dati da www.wfactory.net

=item B<boingtv>  - prende i dati da www.boingtv.it

=item B<sitcom1>  - prende i dati da www.sitcom1.it

=back

Per prima cosa esegui B<tv_grab_it --configure> per scegliere quali
canali vuoi scaricare. In seguito lE<39>esecuzione di B<tv_grab_it> 
senza opzioni manderE<39> sullo standard output i dati in formato XML.

B<--configure> Chiede quali canali scaricare e scrive il file di configurazione.

B<--config-file FILE> Imposta il nome del file di configurazione, di 
default eE<39> B<~/.xmltv/tv_grab_it.conf>. Questo file viene scritto
usando B<--configure> e letto durante il grabbing.

B<--gui OPTION> Usa questa opzione per abilitare una interfaccia grafica.
OPTION puE<39> essere 'Tk', oppure lasciato in bianco per la scelta migliore.
Altri valori possibili per OPTION sono: 'Term' per un terminale normale
(default) e 'TermNoProgressBar' per disabilitare lE<39>uso di XMLTV::ProgressBar.

B<--output FILE> scrive in questo file invece che sullo standard output.

B<--days N> prende dati per N giorni. Di default eE<39> 7.

B<--offset N> parte da N giorni in poi. Normalmente parte da oggi.

B<--quiet> non usa i messaggi di avanzamento che normalmente vengono scritti su standard error.

B<--slow> scarica piuE<39> dettagli (trame, attori...). Questo vuol dire scaricare un nuovo file per ogni programma, quindi di default eE<39> disabilitato per risparmiare tempo.

B<--cache-slow> Se usi lE<39>opzione --cache per accelerare le cose se usi il programma piuE<39> volte alla settimana, con questa opzione verranno messe in cache solo le informazioni --slow, cosiE<39> non dovresti perdere cambiamenti nei palinsesti.

B<--verbose> scrive piuE<39> informazioni su quello che sta facendo il programma, utile per il debugging.

B<--errors-in-xml> scrive gli errori sotto forma di programmi nel file XML, cosiE<39> possono essere visti nel tuo frontend preferito oltre che in STDERR.

B<--backend> imposta la sorgente da utilizzare. Vedi gli esempi.

=head1 ATTENZIONE

Se usi --quiet dovresti usare anche --errors-in-xml, o non avrai nessun
avvertimento per eventuali errori. A differenza delle versioni precedenti,
inoltre, se il grabber non riesce a scaricare nessun dato ritorna un
file XML vuoto (o opzionalmente con solo i warning).

La qualitaE<39> dei dati delle varie sorgenti cambia molto. Per esempio, mytv
era molto semplice, ma completa e usava la minor banda possibile. Skytv ha molti
canali, ma i dati non sono molto buoni a meno che non si usi lE<39>opzione --slow, 
(ed in quel caso serve MOLTO tempo). wfactory eE<39> tutto sommato un buon sito se 
non hai bisogno di tutto il pacchetto sky.

=head1 ESEMPI

=over 

=item tv_grab_it --backend mtvit --configure

configura tv_grab_it usando solo la sorgente mtvit

=item tv_grab_it --backend skylife,wfactory --days 1

prende solo un giorno di dati e utilizza un ordine diverso da quello di default (si sarebbe potuto scrivere anche cosiE<39>: --backend skylife --backend wfactory)

=item tv_grab_it --cache --slow --days 3

prende tutti i dati per i prossimi tre giorni ed usa una cache su disco (sempre raccomandabile).

=back

=head1 UTILIZZO CONSIGLIATO

=over 

=item tv_grab_it --cache --slow --cache-slow --errors-in-xml


=head1 VEDERE ANCHE

L<xmltv(5)>.

=head1 AUTORE

Davide Chiarini, davide.chiarini@gmail.com

puoi anche trovare varie informazioni nel forum http://lnx.htpcpoint.it/forum/

=cut

#default language for warnings set at install time
my $DEF_LANG = 'ita';
######################################################################
# initializations
use warnings;
use strict;

use XMLTV::Version '$Id: tv_grab_it.in,v 1.59 2008/01/15 18:13:07 mnbjhguyt Exp $';
use XMLTV::Capabilities qw/baseline manualconfig cache/;
use XMLTV::Description 'Italy';
use XMLTV::Supplement qw/GetSupplement/;
use HTML::Entities;
use HTML::Parser;
use URI::Escape;
use Getopt::Long;
use Date::Manip;
use Memoize;
use XMLTV;
use XMLTV::Memoize;
use XMLTV::Ask;
use XMLTV::Config_file;
use XMLTV::ProgressBar;
use XMLTV::DST;
use XMLTV::Get_nice;
use XMLTV::Mode;

#use XML::Simple;

use XMLTV::Usage <<END
$0: get Italian television listings in XMLTV format
To configure: $0 --configure [--config-file FILE]
To grab listings: $0 [--config-file FILE] [--output FILE] [--days N]
        [--offset N] [--quiet] [--slow] [--verbose] [--backend]
        [--errors-in-xml] [--cache-slow]
To list available channels: $0 [--output FILE] [--quiet] --list-channels
To show capabilities: $0 --capabilities
To show version: $0 --version
END
  ;

# Use Log::TraceMessages if installed.
BEGIN {
    eval { require Log::TraceMessages };
    if ($@) {
    *t = sub {};
    *d = sub { '' };
    }
    else {
    *t = \&Log::TraceMessages::t;
    *d = \&Log::TraceMessages::d;
    Log::TraceMessages::check_argv();
    }
}

#max days on the server
my $MAX_DAYS=7;

# default language
my $LANG="it";
my $date_today = UnixDate("today", '%Y-%m-%d');

#searchch is experimental
#my @default_backends = ('skylife', 'mtvit', 'wfactory', 'searchch', 'boingtv', 'sitcom1');
my @default_backends = ('skylife', 'mtvit', 'wfactory', 'boingtv', 'sitcom1');


my %channels; #to store display names

# backend configurations
my %backend_info
  = ( 
     'wfactory' =>
     { domain => 'wfactory.net',
       base_chan => 'http://www.wfactory.net/showcase/programmi.jsp?FRM_SEARCH_DATE='.$date_today.'&FRM_SEARCH_START_TIME=16%3A00&FRM_SEARCH_TYPE=%25%25&FRM_SEARCH_PACK=%25%25&FRM_SEARCH_CHANNEL=%25%25&canale=%2FTelevisione&x=4&y=4',
       base_data => 'http://www.wfactory.net/showcase/search_channel.jsp?',
       rturl     => "http://www.wfactory.net/showcase/programmi.jsp",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub   => \&wfactory_fetch_data,
       channel_list_sub => \&wfactory_get_channels_list,
     },

     'skylife' =>
     { domain => 'guidatv.sky.it',
       base_chan => 'http://guidatv.sky.it/static/epg/timeline/data/',
       base_data => 'http://guidatv.sky.it/static/epg/timeline/export/',
       rturl => "http://guidatv.sky.it/",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&skylife_fetch_data,
       channel_list_sub => \&skylife_get_channels_list,
     },

     'boingtv' =>
     { domain => 'boingtv.it',
       base_chan => 'http://www.boingtv.it/palinsesto/dati/palinsesto.xml',
       base_data => 'http://www.boingtv.it/palinsesto/dati/palinsesto.xml',
       rturl => "http://www.boingtv.it/palinsesto/dati/palinsesto.xml",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&boingtv_fetch_data,
       channel_list_sub => \&boingtv_get_channels_list,
     },

     'sitcom1' =>
     { domain => 'sitcom1.it',
       base_chan => 'http://www.sitcom1.it/guidatv.asp',
       base_data => 'http://www.sitcom1.it/guidatv.asp',
       rturl => "http://www.sitcom1.it/guidatv.asp",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&sitcom1_fetch_data,
       channel_list_sub => \&sitcom1_get_channels_list,
     },

     'searchch' =>
     { domain => 'tv.search.ch',
       base_chan => 'http://tv.search.ch/programm/',
       base_data => 'http://tv.search.ch/programm/station/detail.php',
       rturl => "http://tv.search.ch/programm/station/detail.php",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&searchch_fetch_data,
       channel_list_sub => \&searchch_get_channels_list,
     },

     'mtvit' =>
     { domain => 'www.mtv.it',
       base_chan => 'http://www.mtv.it/tv/palinsesto/index.asp',
       base_data => 'http://www.mtv.it/tv/palinsesto/index.asp',
       rturl => "http://www.mtv.it/tv/palinsesto/index.asp",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&mtvit_fetch_data,
       channel_list_sub => \&mtvit_get_channels_list,
     },


    );

######################################################################
# Get options, including undocumented --cache option.
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux') # cache on disk
  or memoize('XMLTV::Get_nice::get_nice_aux')               # cache in memory
  or die "cannot memoize 'XMLTV::Get_nice::get_nice_aux': $!";

my ($opt_days,
    $opt_offset,
    $opt_help,
    $opt_output,
    $opt_slow,
    $opt_verbose,
    $opt_configure,
    $opt_config_file,
    $opt_gui,
    $opt_quiet,
    $opt_errors_in_xml,
    @opt_backends,
    $opt_list_channels,
	$opt_cache_slow,
   );

# server only holds 7 days, so if there is an offset days must be
# opt_days-offset or less.

$opt_offset = 0;   # default
$opt_quiet  = 0;   # default
$opt_slow   = 0;   # default
$opt_verbose = 0;  # default

GetOptions('days=i'       => \$opt_days,
       'offset=i'         => \$opt_offset,
       'help'             => \$opt_help,
       'configure'        => \$opt_configure,
       'config-file=s'    => \$opt_config_file,
       'gui:s'            => \$opt_gui,
       'output=s'         => \$opt_output,
       'quiet'            => \$opt_quiet,
       'slow'             => \$opt_slow,
       'verbose'          => \$opt_verbose,
       'errors-in-xml'    => \$opt_errors_in_xml,
       'backend=s'        => \@opt_backends,
       'list-channels'    => \$opt_list_channels,
       'cache-slow'       => \$opt_cache_slow,
      )
  or usage(0);

die ($DEF_LANG eq 'eng' ? 
         "number of days (--days) must not be negative. You gave: $opt_days\n" :
         "il numero di giorni (--days) non puo' essere negativo. Hai specificato: $opt_days\n")
  if (defined $opt_days && $opt_days < 0);

die ($DEF_LANG eq 'eng' ?
        "offset days (--offset) must not be negative. You gave: $opt_offset\n" :
        "l'intervallo di partenza (--offset) non puo' essere negativo. Hai specificato: $opt_offset\n")
  if ($opt_offset < 0);
usage(1) if $opt_help;

if ($opt_quiet) {
    $opt_verbose = 0;
}

$opt_days = $opt_days || $MAX_DAYS;

$opt_slow = 1 if ($opt_cache_slow);

my $mode = XMLTV::Mode::mode('grab',
			     $opt_list_channels => 'list-channels',
			     $opt_configure => 'configure');

# parse the --backend option
@opt_backends = split(/,/,join(',',@opt_backends)); #we allow both multiple --backend and --backend=name1,name2

my @backends = ();
foreach (@opt_backends) {
    if (defined $backend_info{$_}) {
        push @backends, $_;
    }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "Unknown backend $_!\n" :
                          "Fonte sconosciuta $_!\n"
                         );
    }
}
unless (@backends) {
    @backends = @default_backends;
    if (@opt_backends) {    #we specified backends but we didn't like them, warn the user
        warn ($DEF_LANG eq 'eng' ?
                        "No good backend specified, falling back on defaults\n" : 
                        "Nessuna fonte corretta specificata, uso i default\n"
                        );
    }
}

XMLTV::Ask::init($opt_gui);

# reads the file channel_ids, which contains the tables to convert 
# between backends' ids and XMLTV ids of channels.
# to support multiple backends i add a ini-style [section] header
# there are two fields: xmltv_id and site_id.

my $str = GetSupplement( "tv_grab_it", "channel_ids" );
my $CHANNEL_NAMES_FILE = "channel_ids";

my (%xmltv_chanid, %seen);
my $line_num = 0;
my $backend;
foreach (split( /\n/, $str )) {
    ++ $line_num;
    tr/\r//d;

    s/#.*//;
    next if m/^\s*$/;

    my $where = "$CHANNEL_NAMES_FILE:$line_num";

    if (/^\[(.*)\]$/) {
        if (defined $backend_info{$1}) {    #esiste la configurazione
            $backend = $1;
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                "Unknown backend $1 in $where\n" :
                                "Fonte sconosciuta $1 in $where\n");
            $backend = undef;
        }
    }
    elsif ($backend) {
        my @fields = split /;/;
        die ($DEF_LANG eq 'eng' ?
                        "$where: wrong number of fields" : 
                        "$where: numero di campi errato")
          if @fields != 2;#3;

        my ($xmltv_id, $site_id) = @fields;

        warn ($DEF_LANG eq 'eng' ?
                          "$where: backend id $site_id for site '$backend' seen already\n" :
                          "$where: fonte con id $site_id per il sito '$backend' gia' visto!\n"
                          )
          if defined $backend_info{$backend}{site_ids}{$xmltv_id};
        $backend_info{$backend}{site_ids}{$xmltv_id}{site_id} = $site_id;
        #$backend_info{$backend}{site_ids}{$xmltv_id}{satellite} = $sat;

        warn ($DEF_LANG eq 'eng' ?
                         "$where: XMLTV_id $xmltv_id for site '$backend' seen already\n" :
                         "$where: XMLTV_id $xmltv_id per il sito '$backend' gia' visto!\n" )
          if $seen{$backend.$xmltv_id}++;
    }
}

# File that stores which channels to download.  Not needed for
# list-channels mode.
#
my $config_file;
unless ($mode eq 'list-channels') {
    $config_file = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_it', $opt_quiet);
}

XMLTV::Config_file::check_no_overwrite($config_file) if $mode eq 'configure';

# Arguments for XMLTV::Writer.
my %w_args;
if (defined $opt_output) {
    die($DEF_LANG eq 'eng' ?
	"cannot give --output with --configure" :
	"non e' possibile specificare --output con --configure")
	if $mode eq 'configure';
    my $fh = new IO::File(">$opt_output");
    die ($DEF_LANG eq 'eng' ?
                 "cannot write to $opt_output: $!" : 
                 "impossibile scrivere su $opt_output") if not defined $fh;
    $w_args{OUTPUT} = $fh;
}
$w_args{encoding} = 'ISO-8859-1';


$line_num = 0;

my $foundchannels;

my $bar = new XMLTV::ProgressBar(($DEF_LANG eq 'eng' ? 'getting list of channels' : 'prendo la lista dei canali'), scalar @backends)
  if not $opt_quiet;
# find list of available channels
foreach $backend (@backends) {
    %{$backend_info{$backend}{channels}} = &{$backend_info{$backend}{channel_list_sub}}($backend_info{$backend}{base_chan});
    $foundchannels+=scalar(keys(%{$backend_info{$backend}{channels}}));

    if (not $opt_quiet) {
        update $bar; 
    }
}
$bar->finish() if (not $opt_quiet);
die ($DEF_LANG eq 'eng' ? "no channels could be found" : "nessun canale trovato") unless ($foundchannels);
warn ($DEF_LANG eq 'eng' ? 
     "VERBOSE: $foundchannels channels found.\n" :
         "VERBOSE: $foundchannels canali trovati.\n") if ($opt_verbose);

######################################################################
# write configuration
if ($mode eq 'configure') {
    open(CONF, ">$config_file") or die ($DEF_LANG eq 'eng' ? 
                                                "cannot write to $config_file: $!" :
                                                "impossibile scrivere su $config_file: $!");

    my %channels;
    foreach $backend (@backends) {
        #faccio un hash con tutti gli id
        foreach (keys %{$backend_info{$backend}{channels}}) {
            $channels{$_} = xmltv_chanid($backend, $_);
        }

        #not used yet
        if ($backend_info{$backend}{needs_login}) {
            say "To get listings on '$backend' you will need a login on the site.\n";
            my $username_wanted = ask_boolean('Do you have a login?', 0);
            if ($username_wanted) {
                $backend_info{$backend}{username} = ask("Username:");
                print CONF "username: $backend:$backend_info{$backend}{username}\n";
            }
        }
    }

    #double reverse to get rid of duplicates
    %channels = reverse %channels;
    %channels = reverse %channels;

    # Ask about each channel.
    my @names = sort keys %channels;
    my @qs = map { ($DEF_LANG eq 'eng' ? "add channel $_?" : "aggiungo il canale $_?") } @names;
    my @want = ask_many_boolean(1, @qs);
    foreach (@names) {
        die if $_ =~ tr/\r\n//;
        my $w = shift @want;
        warn("cannot read input, stopping channel questions"), last
          if not defined $w;
        # No need to print to user - XMLTV::Ask is verbose enough.

        # Print a config line, but comment it out if channel not wanted.
        print CONF '#' if not $w;
        print CONF "channel ".$channels{$_}." # $_\n";
    }

    close CONF or warn ($DEF_LANG eq 'eng' ? 
                                "cannot close $config_file: $!" : 
                                "impossibile chiudere $config_file: $!");
    say(($DEF_LANG eq 'eng' ? "Finished configuration." : "Configurazione terminata."));

    exit();
}

# Not configuring, must be writing some XML.
my $w = new XMLTV::Writer(%w_args);

my $source_info_str = join ",", map {'http://'.$backend_info{$_}{domain}} @backends;
my $source_data_str = join ",", map {$backend_info{$_}{rturl}} @backends;

$w->start({ 'source-info-url'     => $source_info_str ,
        'source-data-url'     => $source_data_str,
        'generator-info-name' => 'XMLTV',
        'generator-info-url'  => 'http://membled.com/work/apps/xmltv/',
        });


my %display_names;
foreach my $back (@backends) {
        foreach (keys %{$backend_info{$back}{site_ids}}) {
                $display_names{$_} = $backend_info{$back}{site_ids}{$_}{site_id};
        }
}

if ($mode eq 'list-channels') {
    # Write all known channels then finish.
    foreach my $xmltv_id (sort keys %display_names) {
		next if not defined $display_names{$xmltv_id};

		my @display_name= [ [ $display_names{$xmltv_id} ] ];

		my @chaninfo;
		if (defined $backend_info{skylife}{site_ids}{$xmltv_id}{channum}) { #abbiamo sia il numero che l'icona; 
			@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ], [ $backend_info{skylife}{site_ids}{$xmltv_id}{channum}]],
						icon => [{src => $backend_info{skylife}{site_ids}{$xmltv_id}{icon}}],
						);

		}

		else {@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ] ]);}

		$w->write_channel({
			id => $xmltv_id,
			@chaninfo
			});
	}
    $w->end;
    exit;
}


######################################################################
# read configuration
my @channels;
$line_num = 0;
foreach (XMLTV::Config_file::read_lines($config_file)) {
    ++ $line_num;
    next if not defined;
    if (/^channel:?\s*(.*\S+)\s*$/) {
          push @channels, $1;
    }
    elsif (/^username:?\s+(\S+):(\S+)/){
        if (defined $backend_info{$1}) {    #esiste la configurazione
            $backend_info{$1}{username} = $2;
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                  "Found username for unknown backend $1 in $config_file\n" : 
                                  "Trovato un nome utente per una fonte sconosciuta $1 in $config_file\n");
        }
    }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "$config_file:$line_num: bad line\n" : 
                          "$config_file:$line_num: linea errata\n");
    }
}


######################################################################
# sort out problem in offset options
if ($opt_offset >= $MAX_DAYS) {
    warn ($DEF_LANG eq 'eng' ?
                  "Day offset too big. No program information will be fetched.\n" :
                  "Intervallo specificato troppo grande. Nessun dato verra' scaricato.\n");
    $opt_offset = 0;
    $opt_days = 0;
}
my $days2get;
if (($opt_days+$opt_offset) > $MAX_DAYS) {
    $days2get=$MAX_DAYS-$opt_offset;
    warn ($DEF_LANG eq 'eng' ?
                 "The server only has info for ".($MAX_DAYS-1)." days from today.\n" : 
                 "Il server ha informazioni solo per ".($MAX_DAYS-1)." giorni da oggi.\n");
    if ($days2get > 1) {
        warn ($DEF_LANG eq 'eng' ? 
                          "You'll get listings for only $days2get days.\n" :
                          "Scarico programmi solo per $days2get giorni.\n");
        }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "You'll get listings for only 1 day.\n" : 
                          "Scarico programmi solo per un giorno.\n");
        }
    }
    else {
        $days2get=$opt_days;
    }
t "will get $days2get days from $opt_offset onwards";


######################################################################
# grabbing listings

foreach my $xmltv_id (@channels) {
    next if not defined $display_names{$xmltv_id};

    my @display_name= [ [ $display_names{$xmltv_id} ] ];

	my @chaninfo;
	if (defined $backend_info{skylife}{site_ids}{$xmltv_id}{channum}) { #abbiamo sia il numero che l'icona; 
		@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ], [ $backend_info{skylife}{site_ids}{$xmltv_id}{channum}]],
			        icon => [{src => $backend_info{skylife}{site_ids}{$xmltv_id}{icon}}],
			        );

	}
	else {@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ] ]);}

    $w->write_channel({
        id => $xmltv_id,
        @chaninfo
        });
}

#make a list of channels and days to grab
my @to_get;
foreach my $day ($opt_offset .. ($days2get + $opt_offset - 1)) {
    foreach my $channel (@channels) {
        push @to_get, [$channel, $day];
    }
}

$bar = new XMLTV::ProgressBar(($DEF_LANG eq 'eng' ? 'getting listings' : 'scarico programmi'), scalar @to_get)
  if not $opt_quiet;

## If we aren't getting any days of program data then clear out the list
## that was created to fetch icons.
#if ($days2get == 0) {@to_get = ();}

foreach (@to_get) {
    my $day     = $_->[1];
    my $channel = $_->[0];

    #this is where i would handle cookies and logins if needed
    warn ($DEF_LANG eq 'eng' ? 
                  "VERBOSE: Grabbing channel $channel, day $day\n" :
                  "VERBOSE: Prendo dati per il canale $channel, giorno $day\n") if ($opt_verbose);
 
    my $error;
    foreach $backend (@backends) {
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Trying with $backend\n" :
                          "VERBOSE: Provo con $backend\n") if ($opt_verbose);

        my @dati; $error = 0;
        ($error, @dati) = &{$backend_info{$backend}{fetch_data_sub}}($channel, $day);

        #TODO different kinds of errors?
        if ($error) {
            warn ($DEF_LANG eq 'eng' ?
                                  "VERBOSE: Error fetching channel $channel day $day with backend $backend\n" :
                                  "VERBOSE: Errore nello scaricare i dati per $channel, giorno $day con $backend\n") if ($opt_verbose);
        } 
        else {
            $w->write_programme($_) foreach @dati;
            last;
        }
    }

    #nessuno ci e' riuscito
    if ($error) {
        #this is an easier way to know about errors if all of our scripts are automated
        if ($opt_errors_in_xml) {
            $w->write_programme(
                {
                    title   => [[($DEF_LANG eq 'eng' ? 'ERROR FETCHING DATA' : 'ERRORE DI SCARICAMENTO DATI'), $LANG]],
                    start   => xmltv_date('00:01', $day),
                    stop    => xmltv_date('23:59', $day),
                    channel => $channel,
                    desc    => [[($DEF_LANG eq 'eng' ?
                                                          "XMLTV couldn't grab data for $channel, day $day. Sorry about that." :
                                                          "XMLTV non e' riuscito a scaricare i dati per $channel, giorno $day. Spiacente."), $LANG]],
                }
            );
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                  "I couldn't fetch data for channel $channel, day $day from any backend!!\n" :
                                  "Non sono riuscito a scaricare i dati per $channel, giorno $day da nessuna fonte!!\n") if (not $opt_quiet);
        }
    }

    update $bar if not $opt_quiet;
}
$w->end;
$bar->finish() if not $opt_quiet;

#####################
# general functions #
#####################

####################################################
# xmltv_chanid
# to handle channels that are not yet in the channel_ids file
sub xmltv_chanid {
    my ($backend, $channel_id) = @_;
    my %chan_ids;

    #reverse id hash
    foreach my $xmltv_id (keys %{$backend_info{$backend}{site_ids}}) {
        my $site_id = $backend_info{$backend}{site_ids}{$xmltv_id}{site_id};
        $chan_ids{$site_id} = $xmltv_id;
        next if (not defined $site_id);
        }

    if (defined $chan_ids{$channel_id}) {
        return $chan_ids{$channel_id};
        }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "***Channel |$channel_id| for '$backend' is not in channel_ids, should be updated.\n" : 
                          "***Il canale |$channel_id| su '$backend' non e' in channel_ids, andrebbe aggiornato.\n"
                          ) unless $opt_quiet;
        $channel_id=~ s/\W//gs;

        #make up an id
        my $id = lc($channel_id).".".$backend_info{$backend}{domain};

    ##update backend info
        #$backend_info{$backend}{site_ids}{$id}{site_id} = $channel_id;
        return $id;
    }


}

##########################################################
# tidy
# decodes entities and removes some illegal chars
sub tidy($) {
    for (my $tmp=shift) {
    s/[\000-\037]//g;   # remove control characters
    s/[\222]/\'/g;      # messed up char
    s/[\224]/\"/g;      # end quote
    s/[\205]/\.\.\./g;  # ... must be something messed up in my regexps?
    s/[\223]/\"/g;      #start quote
    s/[\221]/\'/g;

    if (s/[\200-\237]//g) {
        if ($opt_verbose){
            warn ($DEF_LANG eq 'eng' ?
                                  "VERBOSE: removing illegal char: |\\".ord($&)."|\n" : 
                                  "VERBOSE: tolgo carattere illegale: |\\".ord($&)."|\n");
         }
    }

    # Remove leading white space
    s/^\s*//;
    # Remove trailing white space
    s/\s*$//;
    return decode_entities($_);
    }
}


####################################################
# xmltv_date
# this returns a date formatted like 20021229121300 CET
# first argument is time (like '14:20')
# second is date offset from today
sub xmltv_date {
    my ($time, $offset) = @_;

    $time =~/([0-9]+?):([0-9]+).*/ or die ($DEF_LANG eq 'eng' ? "bad time $time" : "strano orario $time");
    my $hour=$1; my $min=$2;

    my $data = &DateCalc("today","+ ".$offset." days");
    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo data') if not defined $data;
    return utc_offset(UnixDate($data, '%Y%m%d').$hour.$min.'00', '+0100');
}


#########################
# wfactory.it functions #
#########################

####################################################
# wfactory_get_channels_list
sub wfactory_get_channels_list {
    my %chan_hash;
    my $base = shift;
    
    my $content;
    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: Getting channel list from $base\n" :
                  "VERBOSE: Scarico la lista dei canali da $base\n") if ($opt_verbose);

        eval { $content = get_nice($base); };
        if ($@) {   #get_nice has died
                warn ($DEF_LANG eq 'eng' ? 
                                          "VERBOSE: Cannot get wfactory's channel list ($base). Site down?\n" :
                                          "VERBOSE: Non sono riuscito a prendere la lista dei canali di wfactory ($base). Il sito non funziona?\n"
                                         )  unless ($opt_quiet);
                return ();
        } 


    my @lines = split /</, $content;

    foreach my $l (@lines) {
        if ($l=~/\Qa href='javascript:ShowChannel(\E(.*?),\"(.*?)\"\)/) {
            $chan_hash{$2}=$1;

            #update backend info, in case this is a new channel not in channel_ids
            #my $xmltv_id = xmltv_chanid('wfactory', $2);
            #$backend_info{wfactory}{site_ids}{$xmltv_id}{site_id} = $1;
        };
    }

    return %chan_hash;
}

####################################################
# wfactory_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub wfactory_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{wfactory}{site_ids}{$xmltv_id}{site_id};

        if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
#   my %chan = reverse %{$backend_info{wfactory}{channels}};
    my %chan = %{$backend_info{wfactory}{channels}};
    my $channel_name  = $backend_info{wfactory}{site_ids}{$xmltv_id}{site_id};
    my $channel_num = $chan{$channel_name}; 
       $channel_name=~s/ /%20/g;

        if (not defined $channel_num) {
                # if we get here it means that the site should have the channel (it's in channel_ids)
                # but for some reason we are missing it's site id (probably the site is down)
                # we return an error so that another backend will by used, if possible
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: \tThis site appears to be down!\n" :
                          "VERBOSE: \tQuesto sito non sembra funzionare!!\n") if ($opt_verbose);
        return (1, ());
        }

    my $date_grab = &DateCalc("today","+ ".$offset." days");


    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo di data') if not defined $date_grab;
    $date_grab = UnixDate($date_grab, '%Y-%m-%d');

    my $url = $backend_info{wfactory}{base_data}.'FRM_CHAN_CHAN='.$channel_num.'&FRM_CHAN_DATE='.$date_grab.'&FRM_CHAN_NAME='.$channel_name;

	#to trick memoize into not caching data with add a string to the url, based on time, with hourly resolution
	#so if we redownload data within 5 minutes (we are within the same run) it comes from the cache
	#but if we download it tomorrow it doesn't.
	#this makes sense if you use the --cache option and you want to cache only the --slow data, to speed up things 
	#when you grab data every two-three days, but you don't want to miss schedule changes
	#this string is ignored by the server
	if ($opt_cache_slow) {
	    my $cachestring = "&pippo=".UnixDate("today","%Y%m%d%H");
		$url.=$cachestring;
	}
    warn ($DEF_LANG eq 'eng' ? 
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend wfactory\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte wfactory\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    $content=~/<td>..\Q<table cellspacing=0 cellpadding=0 border=0 width='100%'>\E(.*?)\Q<\/table>\E/s;
    $content = $1;

    #split and parse
    my @lines = split /\n/, $content;# shift @lines;
    while ( my ($l1, $l2) = splice(@lines, 1, 2)) {
        next unless ($l2);

        my %programme = ();
        my ($title, $time_start, $time_end, $id);

        $l1 =~ /bottom\'>(.*)<\/td>\E/;
        ($time_start, $time_end) = split /-/, $1;

        $l2 =~ /ShowPopUp\((.*)\)'>(.*?)</;
        ($id, $title) = ($1, $2);

        my $past_midnight = 0;
        $past_midnight = 1 if ($time_end < $time_start); #they can work as decimals, 0.32 < 23.44

        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn ($DEF_LANG eq 'eng' ? 'no title found, skipping programme' : 'titolo non trovato, salto');
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn ($DEF_LANG eq 'eng' ? "no start time for title $title, skipping programme" : "nessun orario di inizio per $title, salto");
            next;
        }
            $time_start=~s/\./:/;
            $programme{start}=xmltv_date($time_start, $offset);
        if (not defined $xmltv_id) {
            warn ($DEF_LANG eq 'eng' ? "no channel for programme $title at $time_start, skipping programme" : "canale non trovato per $title alle $time_start, salto");
            next;
        }

        $programme{channel}=$xmltv_id;
        $time_end=~s/\./:/;

        $programme{stop}=xmltv_date($time_end, $offset + $past_midnight);

        if ($opt_slow) {
            wfactory_fetch_data_slow($id, \%programme);
        }

        #put info in array
        push @programmes, {%programme};
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

sub wfactory_fetch_data_slow {
    my ($id, $programme) = @_;

    my $content;
    my $url = 'http://www.wfactory.net/showcase/showtread.jsp?ID='.$id;

    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: getting --slow data from $url" :
                  "VERBOSE: scarico dati --slow da $url") if ($opt_verbose); 
    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: there was some error!" :
                          "VERBOSE: c'e' stato un errore!") if ($opt_verbose);
        return 0;
    }

    if ($content=~/\Q<div name='descript'\E.*?>(.*?)</) {
        my $desc = $1;
        $desc=~s/^\s+//; $desc=~s/\s+$//;
        $programme->{desc}=[[tidy($desc), $LANG] ] if ($desc ne '');
    }

    if ($content=~/\Q<img src='\E(.*?)=yes\'/) {
        #TODO check &amp;
        $programme->{icon}= [{src => 'http://www.wfactory.net/showcase/'.$1.'=yes'}];
    }

    if ($content=~/\Q<!---->\E(.*?)<\/td>/s) {
        my $linetemp=$1;
        $linetemp=~s/^\s+//; $linetemp=~s/\s+$//;
        $linetemp=~/(.*)\Q&nbsp;\E(.*)\Q&nbsp;\E(.*)/;
        my ($category, $year, $country) = ($1, $2, $3); 
        
        $programme->{date} = $year if (defined $year and $year ne '');
        $programme->{country} = [[$country, $LANG]] if (defined $country and $country ne '');
	
	#it looks like category is used only in movies
	if (defined $category and $category ne '') {
	    $programme->{category}=[['Film', $LANG ]];
	    push (@{$programme->{category}}, [tidy($category), $LANG ]);
	}
	
#       warn "|$category|$year|$country\n";
    }

    my @lines = split /\n/, $content;
    foreach my $l (@lines) {
        $l = tidy($l);

        if ($l=~/^<b>(.*?)<\/b>(.*)/mgi) {
            my ($cat, $val) = ($1, $2);
            $val=~s/<br>$//;
            for ($cat){
                /Regia:/ && do {
                    my @directors = split /, /, $val;
                    foreach $a (@directors) {
                        $a=~s/^\s+//; $a=~s/\s+$//;
                        push @{$programme->{credits}->{director}}, $a;
                        }
                    last;
                    };

                /Con:|Con la voce di:/ && do {
                    my @cast = split /,/, $val;
                    foreach (@cast) {
                        s/^\s+//; s/\s+$//;
                        (push @{$programme->{credits}->{actor}}, $_);
                        }
                    last;
                    };

                /Condotto da:|A cura di:/ && do {
                    my @cast = split /,/, $val;
                    foreach (@cast) {
                        s/^\s+//; s/\s+$//;
                        (push @{$programme->{credits}->{presenter}}, $_);
                        }
                    last;
                    };

                /Episodio:/ && do {
                    $val=~s/^\s+//; $val=~s/\s+$//;
                    $programme->{'sub-title'}=[[$val, $LANG] ];
                    last;
                    };

                warn ($DEF_LANG eq 'eng' ? 
                                          "Don't know what |$cat|$val| is\n" : 
                                          "Non conosco |$cat|$val|\n");
            }
        }
    }
}


########################
# boingtv.it functions #
########################

#########################################################
# boingtv_get_channels_list
# since this site only has one channel this is a fake sub
sub boingtv_get_channels_list {
    my %chan_hash = ( 'boingtv' ,'www.boingtv.it');

    return %chan_hash;
}

####################################################
# boingtv_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub boingtv_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{boingtv}{site_ids}{$xmltv_id}{site_id};

    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
    # very strange site: only has data till next sunday. if the offset it's too big we return an empty array 
	# but we don't return an error
	my $day_of_week = UnixDate("today", '%w'); #1 (Monday) to 7 (Sunday)
	if ($day_of_week + $offset > 7) {
        return (0, ());
	}

    my $date_grab = &DateCalc("today","+ ".$offset." days");
    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo di data') if not defined $date_grab;
    $date_grab = UnixDate($date_grab, '%Y%m%d');

    my $url = $backend_info{boingtv}{base_data};

	warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend boingtv\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte boingtv\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    my @lines = split /\n/, $content;

    #split the lines
    foreach my $line (@lines) {
		next unless $line=~/^<EVENT/;
		$line=~/timestamp="(.*?)".*name="(.*?)".*description="(.*?)"/;

        my %programme = ();
        my ($title, $time_start, $description) = ($2, $1, $3);

        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn 'no title found, skipping programme';
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn "no start time for title $title, skipping programme";
            next;
        }

		#dobbiamo buttare via quello che non ci interessa
		next unless ($time_start=~/^$date_grab/);

		$programme{desc}=[[tidy($description), $LANG] ] if ($description ne '');
        $programme{start}=$time_start;#xmltv_date($time_start, $offset + $past_midnight);
        $programme{channel}=$xmltv_id;

        #put info in array
        push @programmes, {%programme};
    }

	if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

########################
# mtv.it functions     #
########################

#########################################################
# mtvit_get_channels_list
# since this site only has one channel this is a fake sub
sub mtvit_get_channels_list {
    my %chan_hash = ( 'mtvit' ,'www.mtv.it');

    return %chan_hash;
}

####################################################
# mtvit_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub mtvit_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{mtvit}{site_ids}{$xmltv_id}{site_id};

    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
    my $grabdate      = UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y:%m:%d');
    my ($anno, $mese, $giorno) = split /:/, $grabdate;

    my $url = $backend_info{mtvit}{base_data}.'?canaleSel=MTV&giorno_guid='.$giorno.'%2F'.$mese.'%2F'.$anno;

    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };

    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend mtvit\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte mtvit\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    $content=~/colonna centrale(.*)colonna destra/s; $content=$1;
    $content=~s/[\n|\r]+//gm;

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    my @lines = split /><strong>/, $content;

    #split the lines
    foreach my $line (@lines) {
	next unless $line=~/TITOLO-LANCIO/;
	$line=~/(.*?) - (.*?)<\/strong> - <STRONG CLASS=\"TITOLO-LANCIO\">(.*?)<\/STRONG><br>(.*?)<br>(.*?)<\/td>/;

        my %programme = ();
        my ($title, $time_start, $time_stop, $description) = ($3, $1, $2, $5);
	my ($time_start2, $time_stop2) = ($time_start, $time_stop);
	$time_start=~s/\./:/ || warn "$line!$time_start!$time_stop\n$1\n$2\n$3\n$4\n$5\n"; 
	$time_stop=~s/\./:/;


        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn 'no title found, skipping programme';
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn "no start time for title $title, skipping programme";
            next;
        }
	
	$programme{desc}=[[tidy($description), $LANG] ] if ($description ne '');
        $programme{start}=xmltv_date($time_start, $offset);

	if ($time_start2 <7 and $time_start2>=0) {
	    $programme{start}=xmltv_date($time_start, $offset + 1);
	    }
	else {
	    $programme{start}=xmltv_date($time_start, $offset);
	}


	if ($time_stop2 <=7 and $time_stop2>=0) { #they can work as decimals, 0.32 < 23.44
	    $programme{stop}=xmltv_date($time_stop, $offset + 1);
	    }
	else {
	    $programme{stop}=xmltv_date($time_stop, $offset);
	}
        $programme{channel}=$xmltv_id;

        #put info in array
        push @programmes, {%programme};
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

########################
# sitcom1.it functions #
########################

#########################################################
# sticom1_get_channels_list
# since this site only has one channel this is a fake sub
sub sitcom1_get_channels_list {
    my %chan_hash = ( 'sitcom1' ,'www.sitcom1.it');

    return %chan_hash;
}

####################################################
# sitcom1_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub sitcom1_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    # date to grab
    my $grabdate      = UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y:%m:%d');
    my ($anno, $mese, $giorno) = split /:/, $grabdate;
    
    # build urls to grab
    my @urls;
    push @urls, $backend_info{sitcom1}{base_data}."?id=1&anno=$anno&mese=$mese&giorno=$giorno";
    push @urls, $backend_info{sitcom1}{base_data}."?id=2&anno=$anno&mese=$mese&giorno=$giorno";    
    push @urls, $backend_info{sitcom1}{base_data}."?id=3&anno=$anno&mese=$mese&giorno=$giorno";    
    push @urls, $backend_info{sitcom1}{base_data}."?id=4&anno=$anno&mese=$mese&giorno=$giorno";    


    my $site_id = $backend_info{sitcom1}{site_ids}{$xmltv_id}{site_id};

    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }
    
    my @programmes = ();
    foreach my $url (@urls) {

	warn ($DEF_LANG eq 'eng' ?
    	          "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

	eval { $content=get_nice($url) };
        if ($@) {   #get_nice has died
            warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend boingtv\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte boingtv\n") if ($opt_verbose);

            # Indicate to the caller that we had problems
            return (1, ());
        } 

        warn "VERBOSE: parsing...\n" if ($opt_verbose);
	
	$content=~s/\n+//igm;
	$content=~s/\r+//igm;	
        $content=~/\Q****** -->\E(.*)\Q<!-- ******\E/;
        $content = tidy($1);

        my @lines = split /height=\"20\"/, $content;

        #split the lines
        foreach my $line (@lines) {
	    next if ($line!~/bgcolor/);
    	    $line=~/10%\".>.(.*?)<\/td.*?href=\"(.*?)\".*?B>(.*?)<\/B.*puntate(.*?)\'.*? >(.*?)<\/a/;
	    #warn "||$1||$2||$3||$4||$5||\n";

            my %programme = ();
            my ($title, $time_start, $subtitle) = ($3, $1, $5);
	    my $category;
	    if ($title=~/Film|Tv movie/i) {
		$category = $title;
		$title = $subtitle;
		$subtitle = undef;
	    }

            # Three mandatory fields: title, start, channel.
            if (not defined $title) {
                warn 'no title found, skipping programme';
                next;
	    }
            $programme{title}=[[tidy($title), $LANG] ];
    	    if (not defined $time_start) {
                warn "no start time for title $title, skipping programme";
        	next;
            }
            $time_start =~/(..).(..)/; my $time_start2 = "$1:$2";
    
            $programme{start}=xmltv_date($time_start2, $offset);
            $programme{'sub-title'}=[[tidy($subtitle), $LANG] ] if defined($subtitle);
    	    $programme{category}=[[tidy($category), $LANG ]] if defined $category;
	    $programme{channel}=$xmltv_id;

            #put info in array
            push @programmes, {%programme};
        }
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}


########################
# skylife.it functions #
########################

####################################################
# skylife_get_channels_list
sub skylife_get_channels_list {
    my %chan_hash;

    my @categories = ('cinema', 'mondo', 'sport', 'news', 'ragazzi', 'intrattenimento', 'primafila', 'hd');

    foreach my $url (@categories) {
	$url = $backend_info{skylife}{base_chan}.$url."_1.xml";
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: Getting channel list from $url\n" :
                          "VERBOSE: Scarico la lista dei canali da $url\n") if ($opt_verbose);
 
        my $content;
        eval { $content = get_nice($url); };

                if ($@) {   #get_nice has died
                        warn ($DEF_LANG eq 'eng' ? 
                                      "VERBOSE: Cannot get skylife's channel list ($url). Site \\n" : 
                                      "VERBOSE: Non sono riuscito a prendere la lista dei canali di skylife ($url). Il sito non funziona?\n"
			     ) unless ($opt_quiet);
                        return ();
                } 


		$content=~/<div class=\"lineatitles\">(.*)<div id=\"contenuti\"/;
		my @canali = split / class=\"canale\">/, $1;
		my $lineacount = 1;
		foreach my $canale (@canali) {
			$canale=~/alt=\"(.*?)\".*src=\"(.*?)\"/;
			next if (not defined $2);
			my $name = tidy($1);
			my $iconurl = $backend_info{skylife}{rturl}.$2;
			my $channum = $2; $channum=~/\/(\d+)\.gif/; $channum=$1;

			#print "$lineacount|$name|$iconurl|$channum\n";


        		$chan_hash{$name} = "$url;$lineacount";
  
        		#update backend info, in case this is a new channel not in channel_ids
        		my $xmltv_id = xmltv_chanid('skylife', $name);
        		$backend_info{skylife}{site_ids}{$xmltv_id}{site_id} = $name;
        		$backend_info{skylife}{site_ids}{$xmltv_id}{icon} = $iconurl;
        		$backend_info{skylife}{site_ids}{$xmltv_id}{channum} = $channum;

			$lineacount++;
  		    }

    }

    return %chan_hash;
}

sub skylife_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{skylife}{site_ids}{$xmltv_id}{site_id};
    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # Indicate to the caller if we have problems (channels disappearing from the site maybe)
    return (1, ()) if (not defined ($backend_info{skylife}{channels}{$site_id}));

    # build url to grab
    my $url = $backend_info{skylife}{base_data}.
			  "canale_export_".$backend_info{skylife}{site_ids}{$xmltv_id}{channum}."_".
			  UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y%m%d').
			  ".xml";
	#il file di domani mi serve per l'ora di fine dell'ultimo programma
    my $url_domani = $backend_info{skylife}{base_data}.
			  "canale_export_".$backend_info{skylife}{site_ids}{$xmltv_id}{channum}."_".
			  UnixDate(&DateCalc("today","+ ".($offset +1)." days"), '%Y%m%d').
			  ".xml";

    my @urls =($url, $url_domani);

    
    #as with other grabber we trick memoize into not caching data
    #however, we do this only for the first day, other days use cache

    my $cachestring = "?pippo=".UnixDate("today","%Y%m%d%H") if ($offset == 0);
	    
    my %prog_to_check = ();

    my $offset2 = 0;
    my $lastid;
    foreach my $url2 (@urls) {
    	my $grabdate      = UnixDate(&DateCalc("today","+ ".($offset + $offset2)." days"), '%Y%m%d');
	$url2.=$cachestring if (($offset+$offset2) == 0);

        warn "VERBOSE: fetching $url2\n"  if ($opt_verbose);

        eval { $content=get_nice($url2) };
        if ($@) {   #get_nice has died
                warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend skylife\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte skylife\n") if ($opt_verbose);

            # Indicate to the caller that we had problems
            return (1, ());
        } 

	warn "VERBOSE: parsing...\n" if ($opt_verbose);

	$content=~/schedule date=\"(.*?)\"/igm;
	#page is not what we expected;
	if ($grabdate ne $1){return (1, ());}

	#split and parse the lines
	my @lines = split /<event id=\"/, $content;

	foreach my $line (@lines) {
		   next if ($line=~/<\?xml version/); #first line

       		   $line=~/(\d+)\"><title>(.*?)<\/title><genre>(.*?)<\/genre><description>(.*?)<\/description><startTime>(.*?):00<\/startTime>/;

			   my ($id, $title, $cat, $desc, $start) = ($1, $2, $3, $4, $5);
			   if ($lastid) {push @{$prog_to_check{$lastid}}, $grabdate." ".$start} #start is lastid stop
			   last if ($url2 eq $url_domani); #ci serve solo il primo programma

			   $prog_to_check{$id} = [$title, $cat, $desc, $grabdate." ".$start];
			   $lastid=$id;
	}
    $offset2++;
    }

    my @programmes = ();
    
    foreach (sort keys %prog_to_check) {
	my %programme = ();
        my ($title, $cat, $desc, $start, $stop) = @{$prog_to_check{$_}};

            $programme{title} = [[tidy($title), $LANG] ];
            $programme{start} = utc_offset(UnixDate($start, '%Y%m%d%H%M').'00', '+0100');
            $programme{stop}  = utc_offset(UnixDate($stop, '%Y%m%d%H%M').'00', '+0100') if $stop;
            $programme{channel} = $xmltv_id;
			my @cats = skylife_split_cats($cat);
			foreach (@cats) {push (@{$programme{category}}, [tidy($_), $LANG ]) if ($_ ne '');}

			skylife_parse_data_slow($desc, \%programme);
			push @programmes, {%programme} if ($start and $title);
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

sub skylife_parse_data_slow {
    my ($content, $programme) = @_;

	my $desc = $content;
    
	my ($cast, $country, $director, $year, $length, $subtitle);

        if ($desc=~/(.*?) - (.*)/) {
	    $subtitle = $1 if ($1 ne '' and $1 ne $programme->{title});
            $desc = $2 if ($2 ne '');
        }

        if ($desc=~/^\'(.*?)\' (.*)/) {
	    $subtitle.= ' - ' if ($subtitle);
    	    $subtitle.= $1 if ($1 ne '' and $1 ne $programme->{title});
            $desc = $2 if ($2 ne '');
        }	


        if ($desc=~/^Regia di (.*?), con (.*?); (.*?) (\d+?) \((\d+) min\)\. (.*)/) {
            $director = $1;
            $cast = $2;
            $country = $3;
            $year = $4;
            my $length = $5;
            $desc = $6 || '';
        }

        if ($desc=~/^Regia di (.*?), con (.*?); (.*?) (\d+?)\. (.*)/) {
            $director = $1;
            $cast = $2;
            $country = $3;
            $year = $4;
            $desc = $5 || '';
        }

        if ($desc=~/^Con ([A-Z].*?)\. (.*)/) {
            $cast = $1;
            $desc = $2 || '';
	}

	#tricky one
        if ($desc=~/^con (.*?)\. (.*)/) {
	   $desc = $2;
	   $cast = $1;
   
	   if ($cast=~/(.*?); (.*)/) {
    		$cast = $1;
		$country = $2;
	    }
        }

	if ($cast) {
	   my $lastcast;
	   ($cast, $lastcast) = split / e /, $cast;
	   my @cast = split /,/, $cast; push @cast, $lastcast if ($lastcast);
           foreach (@cast) {
                s/^\s+//; s/\s+$//;
                (push @{$programme->{credits}->{actor}}, $_);
            }
	}
	
    $content=~s/[\n|\r]+//gm;	

    $programme->{length}= $length*60 if ($length);
    $programme->{date}= $year if ($year);
    $programme->{'sub-title'}=[[$subtitle, $LANG] ] if ($subtitle);	
    push @{$programme->{credits}->{director}}, $director if ($director);
    push (@{$programme->{country}}, [$country, $LANG]) if ($country);
    $programme->{desc}=[[tidy($desc), $LANG ]] if ($desc ne '');

}

sub skylife_split_cats {
  my $text = shift;
  my @res;
  
  if ($text=~/(Intrattenimento|Informazione)(.*)/) {@res=(tidy($2), tidy($1));}
    elsif ($text=~/(Non Definito|Mondo e Tendenze|Ragazzi e Musica|Altri Programmi)(.*)/) {@res=(tidy($2));}
    elsif ($text=~/(Film|Sport)(.*)/) {@res=(tidy($1), tidy($2));}
    else {  
	warn "--------------->Aggiungere categoria:|$text|\n";
	@res=(tidy($text));
    }
  return @res;
}

# everything below this is experimental
########################
# search.ch  functions #
########################

####################################################
# searchch_get_channels_list
sub searchch_get_channels_list {
    my %chan_hash;

    my $url = $backend_info{searchch}{base_chan};
    warn ($DEF_LANG eq 'eng' ?
                      "VERBOSE: Getting channel list from $url\n" :
                      "VERBOSE: Scarico la lista dei canali da $url\n") if ($opt_verbose);
 
    my $content;
    eval { $content = get_nice($url); };

    if ($@) {   #get_nice has died
            warn ($DEF_LANG eq 'eng' ? 
                      "VERBOSE: Cannot get search.ch's channel list ($url). Site down?\n" : 
                      "VERBOSE: Non sono riuscito a prendere la lista dei canali di search.ch ($url). Il sito non funziona?\n"
			) unless ($opt_quiet);
                    return ();
            } 

    my @lines = split /\n/, $content;
    foreach my $line (@lines) {
      if ($line=~/station\/detail.php\?id=(\d+)\".*?und Tagesprogramm zu (.*?)\"/) {
         my ($channum, $channame)=(tidy($1), tidy($2));
         next if (not defined $channum or not defined $channame);# or $channame eq 'TV-Sender' or $channame eq '--------------');
         # 	next if ($channame!~/tsi/igm);
         
         $url = $backend_info{searchch}{base_data}.'?id='.$channum;
         my $iconurl = '/img/stationlogos_id/'.$channum.'.gif';
         $chan_hash{$channame} = "$url";

         my $xmltv_id = xmltv_chanid('searchch', $channame);
         $backend_info{searchch}{site_ids}{$xmltv_id}{site_id} = $channame;
         $backend_info{searchch}{site_ids}{$xmltv_id}{icon} = $iconurl;
         $backend_info{searchch}{site_ids}{$xmltv_id}{channum} = $channum;
         }      
    }

    return %chan_hash;
}

sub searchch_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;
    
    my %langs = (
       'Deutsch' => 'de',
       'Franzsisch' => 'fr',
       'Italienisch' => 'it',
       'Englisch' => 'en',
    );
    
    my $sprache = 'error!';

    my $site_id = $backend_info{searchch}{site_ids}{$xmltv_id}{site_id};
    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
    my $date  = UnixDate(&DateCalc("today","+ ".($offset)." days"), '%Y-%m-%d');
    my $url = $backend_info{searchch}{base_data}.'?id='.$backend_info{searchch}{site_ids}{$xmltv_id}{channum}.'&day='.$date;

    #to trick memoize into not caching data with add a string to the url, based on time, with hourly resolution
    #so if we redownload data within 5 minutes (we are within the same run) it comes from the cache
    #but if we download it tomorrow it doesn't.
    #this makes sense if you use the --cache option and you want to cache only the --slow data, to speed up things 
    #when you grab data every two-three days, but you don't want to miss schedule changes
    #this string is ignored by the server
    
    my $cachestring = "&pippo=".UnixDate("today","%Y%m%d%H");

    my $grabdate      = UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y%m%d');
    my $tomorrowdate      = UnixDate(&DateCalc("today","+ ".($offset+1)." days"), '%Y%m%d');    
	    
    my @prog_to_check = ();

    my $offset2 = 0;
    my $starttime = ParseDate("$grabdate 00:00");
    die 'date calculation failed' if (! $starttime);

    $url.=$cachestring if ($opt_cache_slow);

    warn "VERBOSE: fetching $url\n"  if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
            warn ($DEF_LANG eq 'eng' ? 
                      "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend skylife\n" :
                      "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte skylife\n") if ($opt_verbose);

            # Indicate to the caller that we had problems
            return (1, ());
        } 

    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    #check language
    $content=~/>Sprache: <\/td><td class="text">(.*?)<\/td>/;
    $sprache = $langs{$1} || 'error!';
    
    #is there data?
    if ($content=~/Es wurden keine Sendungen gefunden/){
       #warn "canale $xmltv_id -> no data!\n";
    }
    else {
      #split and parse the lines
      $content=~/<table width=\"100%\" cellpadding=\"0\" cellspacing=\"0\" border=\"0\">(.*?)<\/table>/s;
      $content=$1;
      $content=~s/[\n|\r]+//isgm;
  
      my @lines = split /<tr>/, $content;
  
      foreach my $line (@lines) {
          $line=~/valign=\"top\">(.*?)<\/td>.*<td width=\"100%\"><a href=\"(.*?)\">(.*?)<\/a>(.*?)<\/td>/;
  	      next if not defined $1;
  	
  	      my ($starttime, $detail, $title, $category) = (tidy($1), tidy($2), tidy($3), tidy($4));
  	
  	      if ($category=~/\((.*?)\)/){$category=$1};
  	      my $starttime2=$starttime; $starttime2=~s/:/./;
  	      if ($starttime2<6) {
  	         $starttime = ParseDate("$tomorrowdate $starttime");
  	         }
  	      else {
      	     $starttime = ParseDate("$grabdate $starttime");	
          }
          push @prog_to_check, [$title, $detail, $starttime, $category];
      }
    }

    my @programmes = ();

    my $parse_date = &DateCalc("today 00:00","+ ".$offset." days");
    my $next_day   = &DateCalc("today 00:00","+ ".($offset + 1)." days");

    foreach (@prog_to_check) {
        my %programme = ();
        my ($title, $detail, $start, $category) = @$_;
	      my $subtitle = undef;
	      
	      if ($title=~/(.*) \((.*?)\)$/) {unless (defined $category) {$title=$1;   $category=$2};}# warn "$xmltv_id tit $title cat  $category ";exit;}
	      if ($title=~/(.*) - (.*)/) {$title=$1; $subtitle=$2;}
        $programme{title} = [[tidy($title), $sprache] ];
        $programme{start} = utc_offset(UnixDate($start, '%Y%m%d%H%M').'00', '+0100');
        $programme{channel} = $xmltv_id;
        $programme{'sub-title'}=[[$subtitle, $sprache] ] if (defined $subtitle);	
        if ($category){
           $category = tr_cat(tidy($category));
           push @{$programme{category}}, [tidy($category), 'de' ]
	}
        #searchch_fetch_data_slow($detail, \%programme) if ($opt_slow);
        push @programmes, {%programme} if ($start and $title);
	   }
  
    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}


sub tr_cat {
    my $cat = shift;
    return "Film" if $cat=~/Abenteuer|Action|Komdie|Krimi|Drama|Fantasy/;
    return "Film" if $cat=~/Geschichte|Horror|Kurzfilm|Liebesfilm|Thriller|Trickfilm|Western/;
    return "Sit Com" if $cat=~/Serie|SERIE|Sitcom/;
    return $cat;
}