#!/usr/bin/perl

eval 'exec /usr/bin/perl  -S $0 ${1+"$@"}'
    if 0; # not running under some shell

#################################################
# Changelog:
# 13/07/2005
# - removed backend lastampa, now using 
#   wfactory.net instead (basically the same)
# - removed backend mytv, site doesn't provide 
#   data anymore. Didn't remove the code since
#   it might come in handy in the future.
# 25/08/2005
# - updated after changes in skytv.it site
# - first test with simple double language messages and docs
# 14/06/2006
# - minor update for changes in skytv.it site (now skylife.it)
# 16/08/2006
# - fixes to skytv
# - skytv now handles categories when using --slow
# 11/01/2007
# - added backend boingtv
# - new option --cache-slow
# 13/02/2007
# - added backend skylife (soon to replace skytv)
# 27/05/2007
# - fixed boingtv.it after site change (thanks Paolo Asioli)
# 02/07/2007
# - fixed skylife after site change (thanks Marco Coli)
# 20/09/2007
# fixes for warnings when in quiet mode
# 06/11/2007
# added backend mtv.it, as skylife strangely doesn't carry it, and wfactory is a stuttering site.
# 08/12/2007
# skylife.it has moved to guidatv.sky.it
# code cleanup
# 15/01/2008
# major optimizations in skylife.it! (thanks Massimo Savazzi)
# 30/06/2008
# better handling of season /episodes after site changes
# now using also the dtd tags episode-num
#################################################
# TODO
# - add more informative errors in xml
#################################################

#pod below is handled at install time
=pod

=head1 NAME

tv_grab_it - Grab TV listings for Italy.

=head1 SYNOPSIS

tv_grab_it --help

tv_grab_it [--config-file FILE] --configure

tv_grab_it [--config-file FILE] [--output FILE]
           [--days N] [--offset N] [--quiet]
           [--slow] [--verbose] [--errors-in-xml]
           [--backend SITE1[,SITE2[,SITE3]]]
	   [--cache-slow]
        

=head1 DESCRIPTION

Output TV listings for several channels available in Italy.
The grabber relies on parsing HTML so it might stop working at any time.
The data comes from different backends. This is to minimize blackouts 
in case of site changes but also to extend the number of channels.
If the grabber canE<39>t find the data with the first backend it will
try the second one, and so on. You can specify your order of preference
using the --backend option.

Currently configured backends are (in default order):

=over

=item B<skylife>  - grabs data from www.skylife.it

=item B<mtvit>    - grabs data from www.mtv.it

=item B<wfactory> - grabs data from www.wfactory.net

=item B<boingtv>  - grabs data from www.boingtv.it

=item B<sitcom1>  - grabs data from www.sitcom1.it

=back

First run B<tv_grab_it --configure> to choose which channels you want
to download. Then running B<tv_grab_it> with no arguments will output
listings in XML format to standard output.

B<--configure> Prompt for which channels, and writes the configuration file.

B<--config-file FILE> Set the name of the configuration file, the
default is B<~/.xmltv/tv_grab_it.conf>.  This is the file written
by B<--configure> and read when grabbing.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of XMLTV::ProgressBar.

B<--output FILE> write to FILE rather than standard output.

B<--days N> Grab N days.  The default is 7.

B<--offset N> Start N days in the future.  The default is to start
from today.

B<--quiet> Suppress the progress messages normally written to standard
error.

B<--slow> Downloads more details (descriptions, actors...). This means
downloading a new file for each programme, so itE<39>s off by default to
save time.

B<--cache-slow> If you use the --cache option to speed up thing when you 
grab data several times a week, using this option you will cache only the
--slow data, so you shouldnE<39>t miss changes in schedules.

B<--verbose> Prints out verbose information useful for debugging.

B<--errors-in-xml> Outputs warnings as programmes in the xml file,
so that you can see errors in your favorite frontend in addition
to the default STDERR. 

B<--backend> Set the backend (or backends) to use. See the examples.

B<--version> Show the version of the grabber.

B<--help> Print a help message and exit.

=head1 CAVEATS

If you use --quiet you should also use --errors-in-xml or you wonE<39>t
be warned about errors. Note also that, as opposed to previous versions,
this grabber doesnE<39>t die if it cannot find any data, but returns an
empty (or optionally containing just warnings) xml file instead.

The backendsE<39> data quality differs a lot. For example, mytv was very
basic, yet complete and uses the least amount of bandwith. Skytv has a
lot of channels, but unless you use it with the --slow option the data
is not very good (and in this case i would be VERY slow). wfactory is a 
good overall site if you donE<39>t need the whole sky package.

=head1 EXAMPLES

=over 

=item tv_grab_it --backend mtvit --configure

configures tv_grab_it using only the backend mtvit

=item tv_grab_it --backend skylife,wfactory --days 1

grabs one day of data overiding the default order (could also be --backend skylife --backend wfactory)

=item tv_grab_it --cache --slow --days 3

grabs the full data for the next three days using the default backend order and using a disk cache.

=back

=head1 RECOMMENDED USAGE

=over 

=item tv_grab_it --cache --slow --cache-slow --errors-in-xml

=head1 SEE ALSO

L<xmltv(5)>.

=head1 AUTHOR

Davide Chiarini, davide.chiarini@gmail.com

you can find some more help at http://lnx.htpcpoint.it/forum/

=cut

#default language for warnings set at install time
my $DEF_LANG = 'eng';
######################################################################
# initializations
use warnings;
use strict;

use XMLTV::Version '$Id: tv_grab_it.in,v 1.60 2008/06/30 15:39:37 mnbjhguyt Exp $';
use XMLTV::Capabilities qw/baseline manualconfig cache/;
use XMLTV::Description 'Italy';
use XMLTV::Supplement qw/GetSupplement/;
use HTML::Entities;
use HTML::Parser;
use URI::Escape;
use Getopt::Long;
use Date::Manip;
use Memoize;
use XMLTV;
use XMLTV::Memoize;
use XMLTV::Ask;
use XMLTV::Config_file;
use XMLTV::ProgressBar;
use XMLTV::DST;
use XMLTV::Get_nice;
use XMLTV::Mode;

#use XML::Simple;

use XMLTV::Usage <<END
$0: get Italian television listings in XMLTV format
To configure: $0 --configure [--config-file FILE]
To grab listings: $0 [--config-file FILE] [--output FILE] [--days N]
        [--offset N] [--quiet] [--slow] [--verbose] [--backend]
        [--errors-in-xml] [--cache-slow]
To list available channels: $0 [--output FILE] [--quiet] --list-channels
To show capabilities: $0 --capabilities
To show version: $0 --version
END
  ;

# Use Log::TraceMessages if installed.
BEGIN {
    eval { require Log::TraceMessages };
    if ($@) {
    *t = sub {};
    *d = sub { '' };
    }
    else {
    *t = \&Log::TraceMessages::t;
    *d = \&Log::TraceMessages::d;
    Log::TraceMessages::check_argv();
    }
}

#max days on the server
my $MAX_DAYS=7;

# default language
my $LANG="it";
my $date_today = UnixDate("today", '%Y-%m-%d');

#searchch is experimental
#my @default_backends = ('skylife', 'mtvit', 'wfactory', 'searchch', 'boingtv', 'sitcom1');
my @default_backends = ('skylife', 'mtvit', 'wfactory', 'boingtv', 'sitcom1');


my %channels; #to store display names

# backend configurations
my %backend_info
  = ( 
     'wfactory' =>
     { domain => 'wfactory.net',
       base_chan => 'http://www.wfactory.net/showcase/programmi.jsp?FRM_SEARCH_DATE='.$date_today.'&FRM_SEARCH_START_TIME=16%3A00&FRM_SEARCH_TYPE=%25%25&FRM_SEARCH_PACK=%25%25&FRM_SEARCH_CHANNEL=%25%25&canale=%2FTelevisione&x=4&y=4',
       base_data => 'http://www.wfactory.net/showcase/search_channel.jsp?',
       rturl     => "http://www.wfactory.net/showcase/programmi.jsp",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub   => \&wfactory_fetch_data,
       channel_list_sub => \&wfactory_get_channels_list,
     },

     'skylife' =>
     { domain => 'guidatv.sky.it',
       base_chan => 'http://guidatv.sky.it/static/epg/timeline/data/',
       base_data => 'http://guidatv.sky.it/static/epg/timeline/export/',
       rturl => "http://guidatv.sky.it/",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&skylife_fetch_data,
       channel_list_sub => \&skylife_get_channels_list,
     },

     'boingtv' =>
     { domain => 'boingtv.it',
       base_chan => 'http://www.boingtv.it/palinsesto/dati/palinsesto.xml',
       base_data => 'http://www.boingtv.it/palinsesto/dati/palinsesto.xml',
       rturl => "http://www.boingtv.it/palinsesto/dati/palinsesto.xml",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&boingtv_fetch_data,
       channel_list_sub => \&boingtv_get_channels_list,
     },

     'sitcom1' =>
     { domain => 'sitcom1.it',
       base_chan => 'http://www.sitcom1.it/guidatv.asp',
       base_data => 'http://www.sitcom1.it/guidatv.asp',
       rturl => "http://www.sitcom1.it/guidatv.asp",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&sitcom1_fetch_data,
       channel_list_sub => \&sitcom1_get_channels_list,
     },

     'searchch' =>
     { domain => 'tv.search.ch',
       base_chan => 'http://tv.search.ch/programm/',
       base_data => 'http://tv.search.ch/programm/station/detail.php',
       rturl => "http://tv.search.ch/programm/station/detail.php",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&searchch_fetch_data,
       channel_list_sub => \&searchch_get_channels_list,
     },

     'mtvit' =>
     { domain => 'www.mtv.it',
       base_chan => 'http://www.mtv.it/tv/palinsesto/index.asp',
       base_data => 'http://www.mtv.it/tv/palinsesto/index.asp',
       rturl => "http://www.mtv.it/tv/palinsesto/index.asp",
       needs_login => 0,
       needs_cookies => 0,
       fetch_data_sub =>   \&mtvit_fetch_data,
       channel_list_sub => \&mtvit_get_channels_list,
     },


    );

######################################################################
# Get options, including undocumented --cache option.
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux') # cache on disk
  or memoize('XMLTV::Get_nice::get_nice_aux')               # cache in memory
  or die "cannot memoize 'XMLTV::Get_nice::get_nice_aux': $!";

my ($opt_days,
    $opt_offset,
    $opt_help,
    $opt_output,
    $opt_slow,
    $opt_verbose,
    $opt_configure,
    $opt_config_file,
    $opt_gui,
    $opt_quiet,
    $opt_errors_in_xml,
    @opt_backends,
    $opt_list_channels,
	$opt_cache_slow,
   );

# server only holds 7 days, so if there is an offset days must be
# opt_days-offset or less.

$opt_offset = 0;   # default
$opt_quiet  = 0;   # default
$opt_slow   = 0;   # default
$opt_verbose = 0;  # default

GetOptions('days=i'       => \$opt_days,
       'offset=i'         => \$opt_offset,
       'help'             => \$opt_help,
       'configure'        => \$opt_configure,
       'config-file=s'    => \$opt_config_file,
       'gui:s'            => \$opt_gui,
       'output=s'         => \$opt_output,
       'quiet'            => \$opt_quiet,
       'slow'             => \$opt_slow,
       'verbose'          => \$opt_verbose,
       'errors-in-xml'    => \$opt_errors_in_xml,
       'backend=s'        => \@opt_backends,
       'list-channels'    => \$opt_list_channels,
       'cache-slow'       => \$opt_cache_slow,
      )
  or usage(0);

die ($DEF_LANG eq 'eng' ? 
         "number of days (--days) must not be negative. You gave: $opt_days\n" :
         "il numero di giorni (--days) non puo' essere negativo. Hai specificato: $opt_days\n")
  if (defined $opt_days && $opt_days < 0);

die ($DEF_LANG eq 'eng' ?
        "offset days (--offset) must not be negative. You gave: $opt_offset\n" :
        "l'intervallo di partenza (--offset) non puo' essere negativo. Hai specificato: $opt_offset\n")
  if ($opt_offset < 0);
usage(1) if $opt_help;

if ($opt_quiet) {
    $opt_verbose = 0;
}

$opt_days = $opt_days || $MAX_DAYS;

$opt_slow = 1 if ($opt_cache_slow);

my $mode = XMLTV::Mode::mode('grab',
			     $opt_list_channels => 'list-channels',
			     $opt_configure => 'configure');

# parse the --backend option
@opt_backends = split(/,/,join(',',@opt_backends)); #we allow both multiple --backend and --backend=name1,name2

my @backends = ();
foreach (@opt_backends) {
    if (defined $backend_info{$_}) {
        push @backends, $_;
    }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "Unknown backend $_!\n" :
                          "Fonte sconosciuta $_!\n"
                         );
    }
}
unless (@backends) {
    @backends = @default_backends;
    if (@opt_backends) {    #we specified backends but we didn't like them, warn the user
        warn ($DEF_LANG eq 'eng' ?
                        "No good backend specified, falling back on defaults\n" : 
                        "Nessuna fonte corretta specificata, uso i default\n"
                        );
    }
}

XMLTV::Ask::init($opt_gui);

# reads the file channel_ids, which contains the tables to convert 
# between backends' ids and XMLTV ids of channels.
# to support multiple backends i add a ini-style [section] header
# there are two fields: xmltv_id and site_id.

my $str = GetSupplement( "tv_grab_it", "channel_ids" );
my $CHANNEL_NAMES_FILE = "channel_ids";

my (%xmltv_chanid, %seen);
my $line_num = 0;
my $backend;
foreach (split( /\n/, $str )) {
    ++ $line_num;
    tr/\r//d;

    s/#.*//;
    next if m/^\s*$/;

    my $where = "$CHANNEL_NAMES_FILE:$line_num";

    if (/^\[(.*)\]$/) {
        if (defined $backend_info{$1}) {    #esiste la configurazione
            $backend = $1;
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                "Unknown backend $1 in $where\n" :
                                "Fonte sconosciuta $1 in $where\n");
            $backend = undef;
        }
    }
    elsif ($backend) {
        my @fields = split /;/;
        die ($DEF_LANG eq 'eng' ?
                        "$where: wrong number of fields" : 
                        "$where: numero di campi errato")
          if @fields != 2;#3;

        my ($xmltv_id, $site_id) = @fields;

        warn ($DEF_LANG eq 'eng' ?
                          "$where: backend id $site_id for site '$backend' seen already\n" :
                          "$where: fonte con id $site_id per il sito '$backend' gia' visto!\n"
                          )
          if defined $backend_info{$backend}{site_ids}{$xmltv_id};
        $backend_info{$backend}{site_ids}{$xmltv_id}{site_id} = $site_id;
        #$backend_info{$backend}{site_ids}{$xmltv_id}{satellite} = $sat;

        warn ($DEF_LANG eq 'eng' ?
                         "$where: XMLTV_id $xmltv_id for site '$backend' seen already\n" :
                         "$where: XMLTV_id $xmltv_id per il sito '$backend' gia' visto!\n" )
          if $seen{$backend.$xmltv_id}++;
    }
}

# File that stores which channels to download.  Not needed for
# list-channels mode.
#
my $config_file;
unless ($mode eq 'list-channels') {
    $config_file = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_it', $opt_quiet);
}

XMLTV::Config_file::check_no_overwrite($config_file) if $mode eq 'configure';

# Arguments for XMLTV::Writer.
my %w_args;
if (defined $opt_output) {
    die($DEF_LANG eq 'eng' ?
	"cannot give --output with --configure" :
	"non e' possibile specificare --output con --configure")
	if $mode eq 'configure';
    my $fh = new IO::File(">$opt_output");
    die ($DEF_LANG eq 'eng' ?
                 "cannot write to $opt_output: $!" : 
                 "impossibile scrivere su $opt_output") if not defined $fh;
    $w_args{OUTPUT} = $fh;
}
$w_args{encoding} = 'ISO-8859-1';


$line_num = 0;

my $foundchannels;

my $bar = new XMLTV::ProgressBar(($DEF_LANG eq 'eng' ? 'getting list of channels' : 'prendo la lista dei canali'), scalar @backends)
  if not $opt_quiet;
# find list of available channels
foreach $backend (@backends) {
    %{$backend_info{$backend}{channels}} = &{$backend_info{$backend}{channel_list_sub}}($backend_info{$backend}{base_chan});
    $foundchannels+=scalar(keys(%{$backend_info{$backend}{channels}}));

    if (not $opt_quiet) {
        update $bar; 
    }
}
$bar->finish() if (not $opt_quiet);
die ($DEF_LANG eq 'eng' ? "no channels could be found" : "nessun canale trovato") unless ($foundchannels);
warn ($DEF_LANG eq 'eng' ? 
     "VERBOSE: $foundchannels channels found.\n" :
         "VERBOSE: $foundchannels canali trovati.\n") if ($opt_verbose);

######################################################################
# write configuration
if ($mode eq 'configure') {
    open(CONF, ">$config_file") or die ($DEF_LANG eq 'eng' ? 
                                                "cannot write to $config_file: $!" :
                                                "impossibile scrivere su $config_file: $!");

    my %channels;
    foreach $backend (@backends) {
        #faccio un hash con tutti gli id
        foreach (keys %{$backend_info{$backend}{channels}}) {
            $channels{$_} = xmltv_chanid($backend, $_);
        }

        #not used yet
        if ($backend_info{$backend}{needs_login}) {
            say "To get listings on '$backend' you will need a login on the site.\n";
            my $username_wanted = ask_boolean('Do you have a login?', 0);
            if ($username_wanted) {
                $backend_info{$backend}{username} = ask("Username:");
                print CONF "username: $backend:$backend_info{$backend}{username}\n";
            }
        }
    }

    #double reverse to get rid of duplicates
    %channels = reverse %channels;
    %channels = reverse %channels;

    # Ask about each channel.
    my @names = sort keys %channels;
    my @qs = map { ($DEF_LANG eq 'eng' ? "add channel $_?" : "aggiungo il canale $_?") } @names;
    my @want = ask_many_boolean(1, @qs);
    foreach (@names) {
        die if $_ =~ tr/\r\n//;
        my $w = shift @want;
        warn("cannot read input, stopping channel questions"), last
          if not defined $w;
        # No need to print to user - XMLTV::Ask is verbose enough.

        # Print a config line, but comment it out if channel not wanted.
        print CONF '#' if not $w;
        print CONF "channel ".$channels{$_}." # $_\n";
    }

    close CONF or warn ($DEF_LANG eq 'eng' ? 
                                "cannot close $config_file: $!" : 
                                "impossibile chiudere $config_file: $!");
    say(($DEF_LANG eq 'eng' ? "Finished configuration." : "Configurazione terminata."));

    exit();
}

# Not configuring, must be writing some XML.
my $w = new XMLTV::Writer(%w_args);

my $source_info_str = join ",", map {'http://'.$backend_info{$_}{domain}} @backends;
my $source_data_str = join ",", map {$backend_info{$_}{rturl}} @backends;

$w->start({ 'source-info-url'     => $source_info_str ,
        'source-data-url'     => $source_data_str,
        'generator-info-name' => 'XMLTV',
        'generator-info-url'  => 'http://membled.com/work/apps/xmltv/',
        });


my %display_names;
foreach my $back (@backends) {
        foreach (keys %{$backend_info{$back}{site_ids}}) {
                $display_names{$_} = $backend_info{$back}{site_ids}{$_}{site_id};
        }
}

if ($mode eq 'list-channels') {
    # Write all known channels then finish.
    foreach my $xmltv_id (sort keys %display_names) {
		next if not defined $display_names{$xmltv_id};

		my @display_name= [ [ $display_names{$xmltv_id} ] ];

		my @chaninfo;
		if (defined $backend_info{skylife}{site_ids}{$xmltv_id}{channum}) { #abbiamo sia il numero che l'icona; 
			@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ], [ $backend_info{skylife}{site_ids}{$xmltv_id}{channum}]],
						icon => [{src => $backend_info{skylife}{site_ids}{$xmltv_id}{icon}}],
						);

		}

		else {@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ] ]);}

		$w->write_channel({
			id => $xmltv_id,
			@chaninfo
			});
	}
    $w->end;
    exit;
}


######################################################################
# read configuration
my @channels;
$line_num = 0;
foreach (XMLTV::Config_file::read_lines($config_file)) {
    ++ $line_num;
    next if not defined;
    if (/^channel:?\s*(.*\S+)\s*$/) {
          push @channels, $1;
    }
    elsif (/^username:?\s+(\S+):(\S+)/){
        if (defined $backend_info{$1}) {    #esiste la configurazione
            $backend_info{$1}{username} = $2;
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                  "Found username for unknown backend $1 in $config_file\n" : 
                                  "Trovato un nome utente per una fonte sconosciuta $1 in $config_file\n");
        }
    }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "$config_file:$line_num: bad line\n" : 
                          "$config_file:$line_num: linea errata\n");
    }
}


######################################################################
# sort out problem in offset options
if ($opt_offset >= $MAX_DAYS) {
    warn ($DEF_LANG eq 'eng' ?
                  "Day offset too big. No program information will be fetched.\n" :
                  "Intervallo specificato troppo grande. Nessun dato verra' scaricato.\n");
    $opt_offset = 0;
    $opt_days = 0;
}
my $days2get;
if (($opt_days+$opt_offset) > $MAX_DAYS) {
    $days2get=$MAX_DAYS-$opt_offset;
    warn ($DEF_LANG eq 'eng' ?
                 "The server only has info for ".($MAX_DAYS-1)." days from today.\n" : 
                 "Il server ha informazioni solo per ".($MAX_DAYS-1)." giorni da oggi.\n");
    if ($days2get > 1) {
        warn ($DEF_LANG eq 'eng' ? 
                          "You'll get listings for only $days2get days.\n" :
                          "Scarico programmi solo per $days2get giorni.\n");
        }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "You'll get listings for only 1 day.\n" : 
                          "Scarico programmi solo per un giorno.\n");
        }
    }
    else {
        $days2get=$opt_days;
    }
t "will get $days2get days from $opt_offset onwards";


######################################################################
# grabbing listings

foreach my $xmltv_id (@channels) {
    next if not defined $display_names{$xmltv_id};

    my @display_name= [ [ $display_names{$xmltv_id} ] ];

	my @chaninfo;
	if (defined $backend_info{skylife}{site_ids}{$xmltv_id}{channum}) { #abbiamo sia il numero che l'icona; 
		@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ], [ $backend_info{skylife}{site_ids}{$xmltv_id}{channum}]],
			        icon => [{src => $backend_info{skylife}{site_ids}{$xmltv_id}{icon}}],
			        );

	}
	else {@chaninfo = ('display-name' => [ [ $display_names{$xmltv_id} ] ]);}

    $w->write_channel({
        id => $xmltv_id,
        @chaninfo
        });
}

#make a list of channels and days to grab
my @to_get;
foreach my $day ($opt_offset .. ($days2get + $opt_offset - 1)) {
    foreach my $channel (@channels) {
        push @to_get, [$channel, $day];
    }
}

$bar = new XMLTV::ProgressBar(($DEF_LANG eq 'eng' ? 'getting listings' : 'scarico programmi'), scalar @to_get)
  if not $opt_quiet;

## If we aren't getting any days of program data then clear out the list
## that was created to fetch icons.
#if ($days2get == 0) {@to_get = ();}

foreach (@to_get) {
    my $day     = $_->[1];
    my $channel = $_->[0];

    #this is where i would handle cookies and logins if needed
    warn ($DEF_LANG eq 'eng' ? 
                  "VERBOSE: Grabbing channel $channel, day $day\n" :
                  "VERBOSE: Prendo dati per il canale $channel, giorno $day\n") if ($opt_verbose);
 
    my $error;
    foreach $backend (@backends) {
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Trying with $backend\n" :
                          "VERBOSE: Provo con $backend\n") if ($opt_verbose);

        my @dati; $error = 0;
        ($error, @dati) = &{$backend_info{$backend}{fetch_data_sub}}($channel, $day);

        #TODO different kinds of errors?
        if ($error) {
            warn ($DEF_LANG eq 'eng' ?
                                  "VERBOSE: Error fetching channel $channel day $day with backend $backend\n" :
                                  "VERBOSE: Errore nello scaricare i dati per $channel, giorno $day con $backend\n") if ($opt_verbose);
        } 
        else {
            $w->write_programme($_) foreach @dati;
            last;
        }
    }

    #nessuno ci e' riuscito
    if ($error) {
        #this is an easier way to know about errors if all of our scripts are automated
        if ($opt_errors_in_xml) {
            $w->write_programme(
                {
                    title   => [[($DEF_LANG eq 'eng' ? 'ERROR FETCHING DATA' : 'ERRORE DI SCARICAMENTO DATI'), $LANG]],
                    start   => xmltv_date('00:01', $day),
                    stop    => xmltv_date('23:59', $day),
                    channel => $channel,
                    desc    => [[($DEF_LANG eq 'eng' ?
                                                          "XMLTV couldn't grab data for $channel, day $day. Sorry about that." :
                                                          "XMLTV non e' riuscito a scaricare i dati per $channel, giorno $day. Spiacente."), $LANG]],
                }
            );
        }
        else {
            warn ($DEF_LANG eq 'eng' ?
                                  "I couldn't fetch data for channel $channel, day $day from any backend!!\n" :
                                  "Non sono riuscito a scaricare i dati per $channel, giorno $day da nessuna fonte!!\n") if (not $opt_quiet);
        }
    }

    update $bar if not $opt_quiet;
}
$w->end;
$bar->finish() if not $opt_quiet;

#####################
# general functions #
#####################

####################################################
# xmltv_chanid
# to handle channels that are not yet in the channel_ids file
sub xmltv_chanid {
    my ($backend, $channel_id) = @_;
    my %chan_ids;

    #reverse id hash
    foreach my $xmltv_id (keys %{$backend_info{$backend}{site_ids}}) {
        my $site_id = $backend_info{$backend}{site_ids}{$xmltv_id}{site_id};
        $chan_ids{$site_id} = $xmltv_id;
        next if (not defined $site_id);
        }

    if (defined $chan_ids{$channel_id}) {
        return $chan_ids{$channel_id};
        }
    else {
        warn ($DEF_LANG eq 'eng' ?
                          "***Channel |$channel_id| for '$backend' is not in channel_ids, should be updated.\n" : 
                          "***Il canale |$channel_id| su '$backend' non e' in channel_ids, andrebbe aggiornato.\n"
                          ) unless $opt_quiet;
        $channel_id=~ s/\W//gs;

        #make up an id
        my $id = lc($channel_id).".".$backend_info{$backend}{domain};

    ##update backend info
        #$backend_info{$backend}{site_ids}{$id}{site_id} = $channel_id;
        return $id;
    }


}

##########################################################
# tidy
# decodes entities and removes some illegal chars
sub tidy($) {
    for (my $tmp=shift) {
    s/[\000-\037]//g;   # remove control characters
    s/[\222]/\'/g;      # messed up char
    s/[\224]/\"/g;      # end quote
    s/[\205]/\.\.\./g;  # ... must be something messed up in my regexps?
    s/[\223]/\"/g;      #start quote
    s/[\221]/\'/g;

    if (s/[\200-\237]//g) {
        if ($opt_verbose){
            warn ($DEF_LANG eq 'eng' ?
                                  "VERBOSE: removing illegal char: |\\".ord($&)."|\n" : 
                                  "VERBOSE: tolgo carattere illegale: |\\".ord($&)."|\n");
         }
    }

    # Remove leading white space
    s/^\s*//;
    # Remove trailing white space
    s/\s*$//;
    return decode_entities($_);
    }
}


####################################################
# xmltv_date
# this returns a date formatted like 20021229121300 CET
# first argument is time (like '14:20')
# second is date offset from today
sub xmltv_date {
    my ($time, $offset) = @_;

    $time =~/([0-9]+?):([0-9]+).*/ or die ($DEF_LANG eq 'eng' ? "bad time $time" : "strano orario $time");
    my $hour=$1; my $min=$2;

    my $data = &DateCalc("today","+ ".$offset." days");
    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo data') if not defined $data;
    return utc_offset(UnixDate($data, '%Y%m%d').$hour.$min.'00', '+0100');
}


#########################
# wfactory.it functions #
#########################

####################################################
# wfactory_get_channels_list
sub wfactory_get_channels_list {
    my %chan_hash;
    my $base = shift;
    
    my $content;
    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: Getting channel list from $base\n" :
                  "VERBOSE: Scarico la lista dei canali da $base\n") if ($opt_verbose);

        eval { $content = get_nice($base); };
        if ($@) {   #get_nice has died
                warn ($DEF_LANG eq 'eng' ? 
                                          "VERBOSE: Cannot get wfactory's channel list ($base). Site down?\n" :
                                          "VERBOSE: Non sono riuscito a prendere la lista dei canali di wfactory ($base). Il sito non funziona?\n"
                                         )  unless ($opt_quiet);
                return ();
        } 


    my @lines = split /</, $content;

    foreach my $l (@lines) {
        if ($l=~/\Qa href='javascript:ShowChannel(\E(.*?),\"(.*?)\"\)/) {
            $chan_hash{$2}=$1;

            #update backend info, in case this is a new channel not in channel_ids
            #my $xmltv_id = xmltv_chanid('wfactory', $2);
            #$backend_info{wfactory}{site_ids}{$xmltv_id}{site_id} = $1;
        };
    }

    return %chan_hash;
}

####################################################
# wfactory_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub wfactory_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{wfactory}{site_ids}{$xmltv_id}{site_id};

        if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
#   my %chan = reverse %{$backend_info{wfactory}{channels}};
    my %chan = %{$backend_info{wfactory}{channels}};
    my $channel_name  = $backend_info{wfactory}{site_ids}{$xmltv_id}{site_id};
    my $channel_num = $chan{$channel_name}; 
       $channel_name=~s/ /%20/g;

        if (not defined $channel_num) {
                # if we get here it means that the site should have the channel (it's in channel_ids)
                # but for some reason we are missing it's site id (probably the site is down)
                # we return an error so that another backend will by used, if possible
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: \tThis site appears to be down!\n" :
                          "VERBOSE: \tQuesto sito non sembra funzionare!!\n") if ($opt_verbose);
        return (1, ());
        }

    my $date_grab = &DateCalc("today","+ ".$offset." days");


    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo di data') if not defined $date_grab;
    $date_grab = UnixDate($date_grab, '%Y-%m-%d');

    my $url = $backend_info{wfactory}{base_data}.'FRM_CHAN_CHAN='.$channel_num.'&FRM_CHAN_DATE='.$date_grab.'&FRM_CHAN_NAME='.$channel_name;

	#to trick memoize into not caching data with add a string to the url, based on time, with hourly resolution
	#so if we redownload data within 5 minutes (we are within the same run) it comes from the cache
	#but if we download it tomorrow it doesn't.
	#this makes sense if you use the --cache option and you want to cache only the --slow data, to speed up things 
	#when you grab data every two-three days, but you don't want to miss schedule changes
	#this string is ignored by the server
	if ($opt_cache_slow) {
	    my $cachestring = "&pippo=".UnixDate("today","%Y%m%d%H");
		$url.=$cachestring;
	}
    warn ($DEF_LANG eq 'eng' ? 
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend wfactory\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte wfactory\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    $content=~/<td>..\Q<table cellspacing=0 cellpadding=0 border=0 width='100%'>\E(.*?)\Q<\/table>\E/s;
    $content = $1;

    #split and parse
    my @lines = split /\n/, $content;# shift @lines;
    while ( my ($l1, $l2) = splice(@lines, 1, 2)) {
        next unless ($l2);

        my %programme = ();
        my ($title, $time_start, $time_end, $id);

        $l1 =~ /bottom\'>(.*)<\/td>\E/;
        ($time_start, $time_end) = split /-/, $1;

        $l2 =~ /ShowPopUp\((.*)\)'>(.*?)</;
        ($id, $title) = ($1, $2);

        my $past_midnight = 0;
        $past_midnight = 1 if ($time_end < $time_start); #they can work as decimals, 0.32 < 23.44

        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn ($DEF_LANG eq 'eng' ? 'no title found, skipping programme' : 'titolo non trovato, salto');
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn ($DEF_LANG eq 'eng' ? "no start time for title $title, skipping programme" : "nessun orario di inizio per $title, salto");
            next;
        }
            $time_start=~s/\./:/;
            $programme{start}=xmltv_date($time_start, $offset);
        if (not defined $xmltv_id) {
            warn ($DEF_LANG eq 'eng' ? "no channel for programme $title at $time_start, skipping programme" : "canale non trovato per $title alle $time_start, salto");
            next;
        }

        $programme{channel}=$xmltv_id;
        $time_end=~s/\./:/;

        $programme{stop}=xmltv_date($time_end, $offset + $past_midnight);

        if ($opt_slow) {
            wfactory_fetch_data_slow($id, \%programme);
        }

        #put info in array
        push @programmes, {%programme};
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

sub wfactory_fetch_data_slow {
    my ($id, $programme) = @_;

    my $content;
    my $url = 'http://www.wfactory.net/showcase/showtread.jsp?ID='.$id;

    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: getting --slow data from $url" :
                  "VERBOSE: scarico dati --slow da $url") if ($opt_verbose); 
    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: there was some error!" :
                          "VERBOSE: c'e' stato un errore!") if ($opt_verbose);
        return 0;
    }

    if ($content=~/\Q<div name='descript'\E.*?>(.*?)</) {
        my $desc = $1;
        $desc=~s/^\s+//; $desc=~s/\s+$//;
        $programme->{desc}=[[tidy($desc), $LANG] ] if ($desc ne '');
    }

    if ($content=~/\Q<img src='\E(.*?)=yes\'/) {
        #TODO check &amp;
        $programme->{icon}= [{src => 'http://www.wfactory.net/showcase/'.$1.'=yes'}];
    }

    if ($content=~/\Q<!---->\E(.*?)<\/td>/s) {
        my $linetemp=$1;
        $linetemp=~s/^\s+//; $linetemp=~s/\s+$//;
        $linetemp=~/(.*)\Q&nbsp;\E(.*)\Q&nbsp;\E(.*)/;
        my ($category, $year, $country) = ($1, $2, $3); 
        
        $programme->{date} = $year if (defined $year and $year ne '');
        $programme->{country} = [[$country, $LANG]] if (defined $country and $country ne '');
	
	#it looks like category is used only in movies
	if (defined $category and $category ne '') {
	    $programme->{category}=[['Film', $LANG ]];
	    push (@{$programme->{category}}, [tidy($category), $LANG ]);
	}
	
#       warn "|$category|$year|$country\n";
    }

    my @lines = split /\n/, $content;
    foreach my $l (@lines) {
        $l = tidy($l);

        if ($l=~/^<b>(.*?)<\/b>(.*)/mgi) {
            my ($cat, $val) = ($1, $2);
            $val=~s/<br>$//;
            for ($cat){
                /Regia:/ && do {
                    my @directors = split /, /, $val;
                    foreach $a (@directors) {
                        $a=~s/^\s+//; $a=~s/\s+$//;
                        push @{$programme->{credits}->{director}}, $a;
                        }
                    last;
                    };

                /Con:|Con la voce di:/ && do {
                    my @cast = split /,/, $val;
                    foreach (@cast) {
                        s/^\s+//; s/\s+$//;
                        (push @{$programme->{credits}->{actor}}, $_);
                        }
                    last;
                    };

                /Condotto da:|A cura di:/ && do {
                    my @cast = split /,/, $val;
                    foreach (@cast) {
                        s/^\s+//; s/\s+$//;
                        (push @{$programme->{credits}->{presenter}}, $_);
                        }
                    last;
                    };

                /Episodio:/ && do {
                    $val=~s/^\s+//; $val=~s/\s+$//;
                    $programme->{'sub-title'}=[[$val, $LANG] ];
                    last;
                    };

                warn ($DEF_LANG eq 'eng' ? 
                                          "Don't know what |$cat|$val| is\n" : 
                                          "Non conosco |$cat|$val|\n");
            }
        }
    }
}


########################
# boingtv.it functions #
########################

#########################################################
# boingtv_get_channels_list
# since this site only has one channel this is a fake sub
sub boingtv_get_channels_list {
    my %chan_hash = ( 'boingtv' ,'www.boingtv.it');

    return %chan_hash;
}

####################################################
# boingtv_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub boingtv_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{boingtv}{site_ids}{$xmltv_id}{site_id};

    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
    # very strange site: only has data till next sunday. if the offset it's too big we return an empty array 
	# but we don't return an error
	my $day_of_week = UnixDate("today", '%w'); #1 (Monday) to 7 (Sunday)
	if ($day_of_week + $offset > 7) {
        return (0, ());
	}

    my $date_grab = &DateCalc("today","+ ".$offset." days");
    die ($DEF_LANG eq 'eng' ? 'date calculation failed' : 'errore di calcolo di data') if not defined $date_grab;
    $date_grab = UnixDate($date_grab, '%Y%m%d');

    my $url = $backend_info{boingtv}{base_data};

	warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend boingtv\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte boingtv\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    my @lines = split /\n/, $content;

    #split the lines
    foreach my $line (@lines) {
		next unless $line=~/^<EVENT/;
		$line=~/timestamp="(.*?)".*name="(.*?)".*description="(.*?)"/;

        my %programme = ();
        my ($title, $time_start, $description) = ($2, $1, $3);

        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn 'no title found, skipping programme';
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn "no start time for title $title, skipping programme";
            next;
        }

		#dobbiamo buttare via quello che non ci interessa
		next unless ($time_start=~/^$date_grab/);

		$programme{desc}=[[tidy($description), $LANG] ] if ($description ne '');
        $programme{start}=$time_start;#xmltv_date($time_start, $offset + $past_midnight);
        $programme{channel}=$xmltv_id;

        #put info in array
        push @programmes, {%programme};
    }

	if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

########################
# mtv.it functions     #
########################

#########################################################
# mtvit_get_channels_list
# since this site only has one channel this is a fake sub
sub mtvit_get_channels_list {
    my %chan_hash = ( 'mtvit' ,'www.mtv.it');

    return %chan_hash;
}

####################################################
# mtvit_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub mtvit_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{mtvit}{site_ids}{$xmltv_id}{site_id};

    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
    my $grabdate      = UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y:%m:%d');
    my ($anno, $mese, $giorno) = split /:/, $grabdate;

    my $url = $backend_info{mtvit}{base_data}.'?canaleSel=MTV&giorno_guid='.$giorno.'%2F'.$mese.'%2F'.$anno;

    warn ($DEF_LANG eq 'eng' ?
                  "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

    eval { $content=get_nice($url) };

    if ($@) {   #get_nice has died
        warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend mtvit\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte mtvit\n") if ($opt_verbose);

        # Indicate to the caller that we had problems
        return (1, ());
    } 

    $content=~/colonna centrale(.*)colonna destra/s; $content=$1;
    $content=~s/[\n|\r]+//gm;

    my @programmes = ();
    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    my @lines = split /><strong>/, $content;

    #split the lines
    foreach my $line (@lines) {
	next unless $line=~/TITOLO-LANCIO/;
	$line=~/(.*?) - (.*?)<\/strong> - <STRONG CLASS=\"TITOLO-LANCIO\">(.*?)<\/STRONG><br>(.*?)<br>(.*?)<\/td>/;

        my %programme = ();
        my ($title, $time_start, $time_stop, $description) = ($3, $1, $2, $5);
	my ($time_start2, $time_stop2) = ($time_start, $time_stop);
	$time_start=~s/\./:/ || warn "$line!$time_start!$time_stop\n$1\n$2\n$3\n$4\n$5\n"; 
	$time_stop=~s/\./:/;


        # Three mandatory fields: title, start, channel.
        if (not defined $title) {
            warn 'no title found, skipping programme';
            next;
        }
            $programme{title}=[[tidy($title), $LANG] ];
        if (not defined $time_start) {
            warn "no start time for title $title, skipping programme";
            next;
        }
	
	$programme{desc}=[[tidy($description), $LANG] ] if ($description ne '');
        $programme{start}=xmltv_date($time_start, $offset);

	if ($time_start2 <7 and $time_start2>=0) {
	    $programme{start}=xmltv_date($time_start, $offset + 1);
	    }
	else {
	    $programme{start}=xmltv_date($time_start, $offset);
	}


	if ($time_stop2 <=7 and $time_stop2>=0) { #they can work as decimals, 0.32 < 23.44
	    $programme{stop}=xmltv_date($time_stop, $offset + 1);
	    }
	else {
	    $programme{stop}=xmltv_date($time_stop, $offset);
	}
        $programme{channel}=$xmltv_id;

        #put info in array
        push @programmes, {%programme};
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

########################
# sitcom1.it functions #
########################

#########################################################
# sticom1_get_channels_list
# since this site only has one channel this is a fake sub
sub sitcom1_get_channels_list {
    my %chan_hash = ( 'sitcom1' ,'www.sitcom1.it');

    return %chan_hash;
}

####################################################
# sitcom1_fetch_data
# 2 parameters: xmltv_id of channel 
#               day offset
# returns an error or an array of data
sub sitcom1_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    # date to grab
    my $grabdate      = UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y:%m:%d');
    my ($anno, $mese, $giorno) = split /:/, $grabdate;
    
    # build urls to grab
    my @urls;
    push @urls, $backend_info{sitcom1}{base_data}."?id=1&anno=$anno&mese=$mese&giorno=$giorno";
    push @urls, $backend_info{sitcom1}{base_data}."?id=2&anno=$anno&mese=$mese&giorno=$giorno";    
    push @urls, $backend_info{sitcom1}{base_data}."?id=3&anno=$anno&mese=$mese&giorno=$giorno";    
    push @urls, $backend_info{sitcom1}{base_data}."?id=4&anno=$anno&mese=$mese&giorno=$giorno";    


    my $site_id = $backend_info{sitcom1}{site_ids}{$xmltv_id}{site_id};

    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }
    
    my @programmes = ();
    foreach my $url (@urls) {

	warn ($DEF_LANG eq 'eng' ?
    	          "VERBOSE: fetching $url\n" :
                  "VERBOSE: scarico $url\n") if ($opt_verbose);

	eval { $content=get_nice($url) };
        if ($@) {   #get_nice has died
            warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend boingtv\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte boingtv\n") if ($opt_verbose);

            # Indicate to the caller that we had problems
            return (1, ());
        } 

        warn "VERBOSE: parsing...\n" if ($opt_verbose);
	
	$content=~s/\n+//igm;
	$content=~s/\r+//igm;	
        $content=~/\Q****** -->\E(.*)\Q<!-- ******\E/;
        $content = tidy($1);

        my @lines = split /height=\"20\"/, $content;

        #split the lines
        foreach my $line (@lines) {
	    next if ($line!~/bgcolor/);
    	    $line=~/10%\".>.(.*?)<\/td.*?href=\"(.*?)\".*?B>(.*?)<\/B.*puntate(.*?)\'.*? >(.*?)<\/a/;
	    #warn "||$1||$2||$3||$4||$5||\n";

            my %programme = ();
            my ($title, $time_start, $subtitle) = ($3, $1, $5);
	    my $category;
	    if ($title=~/Film|Tv movie/i) {
		$category = $title;
		$title = $subtitle;
		$subtitle = undef;
	    }

            # Three mandatory fields: title, start, channel.
            if (not defined $title) {
                warn 'no title found, skipping programme';
                next;
	    }
            $programme{title}=[[tidy($title), $LANG] ];
    	    if (not defined $time_start) {
                warn "no start time for title $title, skipping programme";
        	next;
            }
            $time_start =~/(..).(..)/; my $time_start2 = "$1:$2";
    
            $programme{start}=xmltv_date($time_start2, $offset);
            $programme{'sub-title'}=[[tidy($subtitle), $LANG] ] if defined($subtitle);
    	    $programme{category}=[[tidy($category), $LANG ]] if defined $category;
	    $programme{channel}=$xmltv_id;

            #put info in array
            push @programmes, {%programme};
        }
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}


########################
# skylife.it functions #
########################

####################################################
# skylife_get_channels_list
sub skylife_get_channels_list {
    my %chan_hash;

    my @categories = ('cinema', 'mondo', 'sport', 'news', 'ragazzi', 'intrattenimento', 'primafila', 'hd');

    foreach my $url (@categories) {
	$url = $backend_info{skylife}{base_chan}.$url."_1.xml";
        warn ($DEF_LANG eq 'eng' ?
                          "VERBOSE: Getting channel list from $url\n" :
                          "VERBOSE: Scarico la lista dei canali da $url\n") if ($opt_verbose);
 
        my $content;
        eval { $content = get_nice($url); };

                if ($@) {   #get_nice has died
                        warn ($DEF_LANG eq 'eng' ? 
                                      "VERBOSE: Cannot get skylife's channel list ($url). Site \\n" : 
                                      "VERBOSE: Non sono riuscito a prendere la lista dei canali di skylife ($url). Il sito non funziona?\n"
			     ) unless ($opt_quiet);
                        return ();
                } 


		$content=~/<div class=\"lineatitles\">(.*)<div id=\"contenuti\"/;
		my @canali = split / class=\"canale\">/, $1;
		my $lineacount = 1;
		foreach my $canale (@canali) {
			$canale=~/alt=\"(.*?)\".*src=\"(.*?)\"/;
			next if (not defined $2);
			my $name = tidy($1);
			my $iconurl = $backend_info{skylife}{rturl}.$2;
			my $channum = $2; $channum=~/\/(\d+)\.gif/; $channum=$1;

			#print "$lineacount|$name|$iconurl|$channum\n";


        		$chan_hash{$name} = "$url;$lineacount";
  
        		#update backend info, in case this is a new channel not in channel_ids
        		my $xmltv_id = xmltv_chanid('skylife', $name);
        		$backend_info{skylife}{site_ids}{$xmltv_id}{site_id} = $name;
        		$backend_info{skylife}{site_ids}{$xmltv_id}{icon} = $iconurl;
        		$backend_info{skylife}{site_ids}{$xmltv_id}{channum} = $channum;

			$lineacount++;
  		    }

    }

    return %chan_hash;
}

sub skylife_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;

    my $site_id = $backend_info{skylife}{site_ids}{$xmltv_id}{site_id};
    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # Indicate to the caller if we have problems (channels disappearing from the site maybe)
    return (1, ()) if (not defined ($backend_info{skylife}{channels}{$site_id}));

    # build url to grab
    my $url = $backend_info{skylife}{base_data}.
			  "canale_export_".$backend_info{skylife}{site_ids}{$xmltv_id}{channum}."_".
			  UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y%m%d').
			  ".xml";
	#il file di domani mi serve per l'ora di fine dell'ultimo programma
    my $url_domani = $backend_info{skylife}{base_data}.
			  "canale_export_".$backend_info{skylife}{site_ids}{$xmltv_id}{channum}."_".
			  UnixDate(&DateCalc("today","+ ".($offset +1)." days"), '%Y%m%d').
			  ".xml";

    my @urls =($url, $url_domani);

    
    #as with other grabber we trick memoize into not caching data
    #however, we do this only for the first day, other days use cache

    my $cachestring = "?pippo=".UnixDate("today","%Y%m%d%H") if ($offset == 0);
	    
    my %prog_to_check = ();

    my $offset2 = 0;
    my $lastid;
    foreach my $url2 (@urls) {
    	my $grabdate      = UnixDate(&DateCalc("today","+ ".($offset + $offset2)." days"), '%Y%m%d');
	$url2.=$cachestring if (($offset+$offset2) == 0);

        warn "VERBOSE: fetching $url2\n"  if ($opt_verbose);

        eval { $content=get_nice($url2) };
        if ($@) {   #get_nice has died
                warn ($DEF_LANG eq 'eng' ? 
                          "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend skylife\n" :
                          "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte skylife\n") if ($opt_verbose);

            # Indicate to the caller that we had problems
            return (1, ());
        } 

	warn "VERBOSE: parsing...\n" if ($opt_verbose);

	$content=~/schedule date=\"(.*?)\"/igm;
	#page is not what we expected;
	if ($grabdate ne $1){return (1, ());}

	#split and parse the lines
	my @lines = split /<event id=\"/, $content;

	foreach my $line (@lines) {
		   next if ($line=~/<\?xml version/); #first line

       		   $line=~/(\d+)\"><title>(.*?)<\/title><genre>(.*?)<\/genre><description>(.*?)<\/description><startTime>(.*?):00<\/startTime>/;

			   my ($id, $title, $cat, $desc, $start) = ($1, $2, $3, $4, $5);
			   if ($lastid) {push @{$prog_to_check{$lastid}}, $grabdate." ".$start} #start is lastid stop
			   last if ($url2 eq $url_domani); #ci serve solo il primo programma

			   $prog_to_check{$id} = [$title, $cat, $desc, $grabdate." ".$start];
			   $lastid=$id;
	}
    $offset2++;
    }

    my @programmes = ();
    
    foreach (sort keys %prog_to_check) {
	my %programme = ();
        my ($title, $cat, $desc, $start, $stop) = @{$prog_to_check{$_}};

            $programme{title} = [[tidy($title), $LANG] ];
            $programme{start} = utc_offset(UnixDate($start, '%Y%m%d%H%M').'00', '+0100');
            $programme{stop}  = utc_offset(UnixDate($stop, '%Y%m%d%H%M').'00', '+0100') if $stop;
            $programme{channel} = $xmltv_id;
			my @cats = skylife_split_cats($cat);
			foreach (@cats) {push (@{$programme{category}}, [tidy($_), $LANG ]) if ($_ ne '');}

			skylife_parse_data_slow($desc, \%programme);
			push @programmes, {%programme} if ($start and $title);
    }

    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}

sub skylife_parse_data_slow {
    my ($content, $programme) = @_;

	my $desc = $content;
    
	my ($cast, $country, $director, $year, $length, $subtitle, $episode, $season);

        if ($desc=~/(.*?)\' Stagione - Ep.(\d+?) - (.*)/) {
		    $season = $1;
		    $episode =$2;
            $desc = $3 if ($3 ne '');
        }

        if ($desc=~/(.*?) - (.*)/) {
		    $subtitle = $1 if ($1 ne '' and $1 ne $programme->{title});
            $desc = $2 if ($2 ne '');
		    
		    if ($subtitle=~/(.*?)\' Stagione/){$season = $1;}
		    if ($subtitle=~/Ep.(\d+)/) {$episode = $1;}
			$subtitle='' if ($season or $episode);
        }


        if ($desc=~/^\'(.*?)\' (.*)/) {
		    $subtitle.= ' - ' if ($subtitle);
    	    $subtitle= $1 if ($1 ne '' and $1 ne $programme->{title});
            $desc = $2 if ($2 ne '');
        }
		
		my $strseason = '';
		$strseason.= 'Stagione '.$season if ($season);
		if ($episode and $season){
			$strseason.= ' Episodio '.$episode ;
		}
		elsif ($episode) {
			$strseason.= 'Episodio '.$episode ;
		}

		if ($strseason and $subtitle){
			$subtitle="$strseason - ".$subtitle ;
		}
		else {
			$subtitle=$strseason;
		};

        if ($desc=~/^Regia di (.*?), con (.*?); (.*?) (\d+?) \((\d+) min\)\. (.*)/) {
            $director = $1;
            $cast = $2;
            $country = $3;
            $year = $4;
            my $length = $5;
            $desc = $6 || '';
        }

        if ($desc=~/^Regia di (.*?), con (.*?); (.*?) (\d+?)\. (.*)/) {
            $director = $1;
            $cast = $2;
            $country = $3;
            $year = $4;
            $desc = $5 || '';
        }

        if ($desc=~/^Con ([A-Z].*?)\. (.*)/) {
            $cast = $1;
            $desc = $2 || '';
	}

	#tricky one
        if ($desc=~/^con (.*?)\. (.*)/) {
	   $desc = $2;
	   $cast = $1;
   
	   if ($cast=~/(.*?); (.*)/) {
    		$cast = $1;
		$country = $2;
	    }
        }

	if ($cast) {
	   my $lastcast;
	   ($cast, $lastcast) = split / e /, $cast;
	   my @cast = split /,/, $cast; push @cast, $lastcast if ($lastcast);
           foreach (@cast) {
                s/^\s+//; s/\s+$//;
                (push @{$programme->{credits}->{actor}}, $_);
            }
	}
	
    $content=~s/[\n|\r]+//gm;	

    $programme->{length}= $length*60 if ($length);
    $programme->{date}= $year if ($year);
    $programme->{'sub-title'}=[[$subtitle, $LANG] ] if ($subtitle);	
    push@{$programme->{'episode-num'}}, [$strseason, 'onscreen'] if ($strseason);	
    push@{$programme->{'episode-num'}}, [(defined $season ? ($season-1) : '').".".(defined $episode ? ($episode-1) : '').".0/1", 'xmltv_ns'] if ($strseason);	

	push @{$programme->{credits}->{director}}, $director if ($director);
    push (@{$programme->{country}}, [$country, $LANG]) if ($country);
    $programme->{desc}=[[tidy($desc), $LANG ]] if ($desc ne '');

}

sub skylife_split_cats {
  my $text = shift;
  my @res;
  
  if ($text=~/(Intrattenimento|Informazione)(.*)/) {@res=(tidy($2), tidy($1));}
    elsif ($text=~/(Non Definito|Mondo e Tendenze|Ragazzi e Musica|Altri Programmi)(.*)/) {@res=(tidy($2));}
    elsif ($text=~/(Film|Sport)(.*)/) {@res=(tidy($1), tidy($2));}
    else {  
	warn "--------------->Aggiungere categoria:|$text|\n";
	@res=(tidy($text));
    }
  return @res;
}

# everything below this is experimental
########################
# search.ch  functions #
########################

####################################################
# searchch_get_channels_list
sub searchch_get_channels_list {
    my %chan_hash;

    my $url = $backend_info{searchch}{base_chan};
    warn ($DEF_LANG eq 'eng' ?
                      "VERBOSE: Getting channel list from $url\n" :
                      "VERBOSE: Scarico la lista dei canali da $url\n") if ($opt_verbose);
 
    my $content;
    eval { $content = get_nice($url); };

    if ($@) {   #get_nice has died
            warn ($DEF_LANG eq 'eng' ? 
                      "VERBOSE: Cannot get search.ch's channel list ($url). Site down?\n" : 
                      "VERBOSE: Non sono riuscito a prendere la lista dei canali di search.ch ($url). Il sito non funziona?\n"
			) unless ($opt_quiet);
                    return ();
            } 

    my @lines = split /\n/, $content;
    foreach my $line (@lines) {
      if ($line=~/station\/detail.php\?id=(\d+)\".*?und Tagesprogramm zu (.*?)\"/) {
         my ($channum, $channame)=(tidy($1), tidy($2));
         next if (not defined $channum or not defined $channame);# or $channame eq 'TV-Sender' or $channame eq '--------------');
         # 	next if ($channame!~/tsi/igm);
         
         $url = $backend_info{searchch}{base_data}.'?id='.$channum;
         my $iconurl = '/img/stationlogos_id/'.$channum.'.gif';
         $chan_hash{$channame} = "$url";

         my $xmltv_id = xmltv_chanid('searchch', $channame);
         $backend_info{searchch}{site_ids}{$xmltv_id}{site_id} = $channame;
         $backend_info{searchch}{site_ids}{$xmltv_id}{icon} = $iconurl;
         $backend_info{searchch}{site_ids}{$xmltv_id}{channum} = $channum;
         }      
    }

    return %chan_hash;
}

sub searchch_fetch_data {
    my ($xmltv_id, $offset) = @_;
    my $content;
    
    my %langs = (
       'Deutsch' => 'de',
       'Französisch' => 'fr',
       'Italienisch' => 'it',
       'Englisch' => 'en',
    );
    
    my $sprache = 'error!';

    my $site_id = $backend_info{searchch}{site_ids}{$xmltv_id}{site_id};
    if (not defined $site_id) {
        warn ($DEF_LANG eq 'eng' ?
                         "VERBOSE: \tThis site doesn't know about $xmltv_id!\n" : 
                         "VERBOSE: \tQuesto sito non sa niente di $xmltv_id!\n" ) if ($opt_verbose);
        return (1, ());
    }

    # build url to grab
    my $date  = UnixDate(&DateCalc("today","+ ".($offset)." days"), '%Y-%m-%d');
    my $url = $backend_info{searchch}{base_data}.'?id='.$backend_info{searchch}{site_ids}{$xmltv_id}{channum}.'&day='.$date;

    #to trick memoize into not caching data with add a string to the url, based on time, with hourly resolution
    #so if we redownload data within 5 minutes (we are within the same run) it comes from the cache
    #but if we download it tomorrow it doesn't.
    #this makes sense if you use the --cache option and you want to cache only the --slow data, to speed up things 
    #when you grab data every two-three days, but you don't want to miss schedule changes
    #this string is ignored by the server
    
    my $cachestring = "&pippo=".UnixDate("today","%Y%m%d%H");

    my $grabdate      = UnixDate(&DateCalc("today","+ ".$offset." days"), '%Y%m%d');
    my $tomorrowdate      = UnixDate(&DateCalc("today","+ ".($offset+1)." days"), '%Y%m%d');    
	    
    my @prog_to_check = ();

    my $offset2 = 0;
    my $starttime = ParseDate("$grabdate 00:00");
    die 'date calculation failed' if (! $starttime);

    $url.=$cachestring if ($opt_cache_slow);

    warn "VERBOSE: fetching $url\n"  if ($opt_verbose);

    eval { $content=get_nice($url) };
    if ($@) {   #get_nice has died
            warn ($DEF_LANG eq 'eng' ? 
                      "VERBOSE: Error fetching $url channel $xmltv_id day $offset backend skylife\n" :
                      "VERBOSE: Errore nello scaricare $url, canale $xmltv_id, giorno $offset, fonte skylife\n") if ($opt_verbose);

            # Indicate to the caller that we had problems
            return (1, ());
        } 

    warn "VERBOSE: parsing...\n" if ($opt_verbose);

    #check language
    $content=~/>Sprache: <\/td><td class="text">(.*?)<\/td>/;
    $sprache = $langs{$1} || 'error!';
    
    #is there data?
    if ($content=~/Es wurden keine Sendungen gefunden/){
       #warn "canale $xmltv_id -> no data!\n";
    }
    else {
      #split and parse the lines
      $content=~/<table width=\"100%\" cellpadding=\"0\" cellspacing=\"0\" border=\"0\">(.*?)<\/table>/s;
      $content=$1;
      $content=~s/[\n|\r]+//isgm;
  
      my @lines = split /<tr>/, $content;
  
      foreach my $line (@lines) {
          $line=~/valign=\"top\">(.*?)<\/td>.*<td width=\"100%\"><a href=\"(.*?)\">(.*?)<\/a>(.*?)<\/td>/;
  	      next if not defined $1;
  	
  	      my ($starttime, $detail, $title, $category) = (tidy($1), tidy($2), tidy($3), tidy($4));
  	
  	      if ($category=~/\((.*?)\)/){$category=$1};
  	      my $starttime2=$starttime; $starttime2=~s/:/./;
  	      if ($starttime2<6) {
  	         $starttime = ParseDate("$tomorrowdate $starttime");
  	         }
  	      else {
      	     $starttime = ParseDate("$grabdate $starttime");	
          }
          push @prog_to_check, [$title, $detail, $starttime, $category];
      }
    }

    my @programmes = ();

    my $parse_date = &DateCalc("today 00:00","+ ".$offset." days");
    my $next_day   = &DateCalc("today 00:00","+ ".($offset + 1)." days");

    foreach (@prog_to_check) {
        my %programme = ();
        my ($title, $detail, $start, $category) = @$_;
	      my $subtitle = undef;
	      
	      if ($title=~/(.*) \((.*?)\)$/) {unless (defined $category) {$title=$1;   $category=$2};}# warn "$xmltv_id tit $title cat  $category ";exit;}
	      if ($title=~/(.*) - (.*)/) {$title=$1; $subtitle=$2;}
        $programme{title} = [[tidy($title), $sprache] ];
        $programme{start} = utc_offset(UnixDate($start, '%Y%m%d%H%M').'00', '+0100');
        $programme{channel} = $xmltv_id;
        $programme{'sub-title'}=[[$subtitle, $sprache] ] if (defined $subtitle);	
        if ($category){
           $category = tr_cat(tidy($category));
           push @{$programme{category}}, [tidy($category), 'de' ]
	}
        #searchch_fetch_data_slow($detail, \%programme) if ($opt_slow);
        push @programmes, {%programme} if ($start and $title);
	   }
  
    if (scalar @programmes) {
        return (0, @programmes);
    }
    else {
        # there is a number of reasons why we could get an empty array.
        # so we return an error 
        return (1, @programmes);
    }
}


sub tr_cat {
    my $cat = shift;
    return "Film" if $cat=~/Abenteuer|Action|Komödie|Krimi|Drama|Fantasy/;
    return "Film" if $cat=~/Geschichte|Horror|Kurzfilm|Liebesfilm|Thriller|Trickfilm|Western/;
    return "Sit Com" if $cat=~/Serie|SERIE|Sitcom/;
    return $cat;
}