1 package NonameTV
::Exporter
::Xmltv
;
10 use DateTime
::Format
::Strptime
;
14 use NonameTV
::Exporter
;
15 use NonameTV
::Language qw
/LoadLanguage/;
16 use NonameTV qw
/norm/;
17 use NonameTV
::Config qw
/ReadConfig/;
18 use ReadEvent qw
/ReadEvent/;
20 use XMLTV
::ValidateFile qw
/LoadDtd ValidateFile/;
22 use NonameTV
::Log qw
/d p w StartLogSection EndLogSection SetVerbosity/;
24 use base
'NonameTV::Exporter';
28 Export data in xmltv format.
33 Show which datafiles are created.
36 Show only fatal errors.
39 Print a list of all channels in xml-format to stdout.
42 Remove any old xmltv files from the output directory.
45 Recreate all output files, not only the ones where data has
48 --channel-group <groupname>
49 Export data only for the channel group specified.
52 Append credits after the last event.
56 $XMLTV::ValidateFile
::REQUIRE_CHANNEL_ID
= 0;
60 my $class = ref($proto) || $proto;
61 my $self = $class->SUPER::new
( @_ );
62 bless ($self, $class);
64 defined( $self->{Encoding
} ) or die "You must specify Encoding.";
65 defined( $self->{DtdFile
} ) or die "You must specify DtdFile.";
66 defined( $self->{Root
} ) or die "You must specify Root";
67 defined( $self->{Language
} ) or die "You must specify Language";
69 $self->{MaxDays
} = 365 unless defined $self->{MaxDays
};
70 $self->{MinDays
} = $self->{MaxDays
} unless defined $self->{MinDays
};
71 $self->{PastDays
} = 8 unless defined $self->{PastDays
};
73 $self->{LastRequiredDate
} =
74 DateTime
->today->add( days
=> $self->{MinDays
}-1 )->ymd("-");
76 $self->{OptionSpec
} = [ qw
/export
-channels remove
-old force
-export append
-credits
78 verbose
+ quiet
+ help
/ ];
80 $self->{OptionDefaults
} = {
81 'export-channels' => 0,
84 'channel-group' => "",
88 'append-credits' => 0,
91 $self->{conf
} = ReadConfig
();
93 LoadDtd
( $self->{DtdFile
} );
95 my $ds = $self->{datastore
};
97 # Load language strings
98 $self->{lngstr
} = LoadLanguage
( $self->{Language
},
99 "exporter-xmltv", $ds );
101 # if KeepXml is set, xml files are not deleted after gzip
102 # (disabled by default)
103 $self->{KeepXml
} = 0 unless defined $self->{KeepXml
};
110 my( $self, $p ) = @_;
111 my $channelgroup = $p->{'channel-group'};
116 Export data
in xmltv
-format with one file per day
and channel
.
121 Generate an xml
-file listing all channels
and their corresponding
125 Remove all data
-files
for dates that have already passed
.
128 Export all data
. Default is to only export data
for batches that
129 have changed since the
last export
.
131 --channel
-group
<groupname
>
132 Export data only
for the channel group specified
.
135 Append credits after the
last event
.
142 SetVerbosity
( $p->{verbose
}, $p->{quiet
} );
144 StartLogSection
( "Xmltv", 0 );
146 if( $p->{'export-channels'} ) {
147 $self->ExportChannelList( $channelgroup );
149 elsif( $p->{'remove-old'} ) {
154 my $update_started = time();
155 my $last_update = $self->ReadState();
157 if( $p->{'append-credits'} ) {
158 $self->{appendcredits
} = 1;
161 if( $p->{'force-export'} ) {
162 $self->FindAll( $todo );
165 $self->FindUpdated( $todo, $last_update );
166 $self->FindUnexportedDays( $todo, $last_update );
169 $self->ExportData( $todo );
171 $self->WriteState( $update_started );
173 EndLogSection
( "Xmltv" );
177 # Find all dates for each channel
182 my $ds = $self->{datastore
};
184 my ( $res, $channels ) = $ds->sa->Sql(
185 "select id from channels where export=1");
187 my $last_date = DateTime
->today->add( days
=> $self->{MaxDays
} -1 );
188 my $first_date = DateTime
->today;
190 while( my $data = $channels->fetchrow_hashref() ) {
191 add_dates
( $todo, $data->{id
},
192 '1970-01-01 00:00:00', '2100-12-31 23:59:59',
193 $first_date, $last_date );
199 # Find all dates that may have new data for each channel.
202 my( $todo, $last_update ) = @_;
204 my $ds = $self->{datastore
};
206 my ( $res, $update_batches ) = $ds->sa->Sql( << 'EOSQL'
207 select channel_id
, batch_id
,
208 min
(start_time
)as min_start
, max
(start_time
) as max_start
211 select id from batches where last_update
> ?
213 group by channel_id
, batch_id
218 my $last_date = DateTime
->today->add( days
=> $self->{MaxDays
} -1 );
219 my $first_date = DateTime
->today;
221 while( my $data = $update_batches->fetchrow_hashref() ) {
222 add_dates
( $todo, $data->{channel_id
},
223 $data->{min_start
}, $data->{max_start
},
224 $first_date, $last_date );
227 $update_batches->finish();
230 # Find all dates that should be exported but haven't been exported
232 sub FindUnexportedDays
{
234 my( $todo, $last_update ) = @_;
236 my $ds = $self->{datastore
};
238 my $days = int( time()/(24*60*60) ) - int( $last_update/(24*60*60) );
239 $days = $self->{MaxDays
} if $days > $self->{MaxDays
};
242 # The previous export was done $days ago.
244 my $last_date = DateTime
->today->add( days
=> $self->{MaxDays
} -1 );
245 my $first_date = $last_date->clone->subtract( days
=> $days-1 );
247 my ( $res, $channels ) = $ds->sa->Sql(
248 "select id from channels where export=1");
250 while( my $data = $channels->fetchrow_hashref() ) {
251 add_dates
( $todo, $data->{id
},
252 '1970-01-01 00:00:00', '2100-12-31 23:59:59',
253 $first_date, $last_date );
264 my $ds = $self->{datastore
};
266 foreach my $channel (keys %{$todo}) {
267 my $chd = $ds->sa->Lookup( "channels", { id
=> $channel } );
269 if( $self->{appendcredits
} ){
270 if( $self->{conf
}->{Site
}->{AppendCopyright
} eq 1 ){
271 $self->{copyright
} = ReadEvent
( $chd->{sched_lang
}, "copyright.txt" );
275 foreach my $date (sort keys %{$todo->{$channel}}) {
276 $self->ExportFile( $chd, $date );
284 my $ds = $self->{datastore
};
286 my $last_update = $ds->sa->Lookup( 'state', { name
=> "xmltv_last_update" },
289 if( not defined( $last_update ) )
291 $ds->sa->Add( 'state', { name
=> "xmltv_last_update", value
=> 0 } );
300 my( $update_started ) = @_;
302 my $ds = $self->{datastore
};
304 $ds->sa->Update( 'state', { name
=> "xmltv_last_update" },
305 { value
=> $update_started } );
308 #######################################################
313 my( $h, $chid, $from, $to, $first, $last ) = @_;
315 my $from_dt = create_dt
( $from, 'UTC' )->truncate( to
=> 'day' );
316 my $to_dt = create_dt
( $to, 'UTC' )->truncate( to
=> 'day' );
318 $to_dt = $last->clone() if $last < $to_dt;
319 $from_dt = $first->clone() if $first > $from_dt;
321 my $first_dt = $from_dt->clone()->subtract( days
=> 1 );
323 for( my $dt = $first_dt->clone();
324 $dt <= $to_dt; $dt->add( days
=> 1 ) ) {
325 $h->{$chid}->{$dt->ymd('-')} = 1;
331 my( $str, $tz ) = @_;
333 my( $year, $month, $day, $hour, $minute, $second ) =
334 ( $str =~ /^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/ );
336 if( defined( $second ) ) {
337 return DateTime
->new(
347 ( $year, $month, $day ) =
348 ( $str =~ /^(\d{4})-(\d{2})-(\d{2})$/ );
350 die( "Xmltv: Unknown time format $str" )
353 return DateTime
->new(
360 #######################################################
362 # Xmltv-specific methods.
367 my( $chd, $date ) = @_;
369 my $section = "Xmltv $chd->{xmltvid}_$date";
371 StartLogSection
( $section, 0 );
375 my $startdate = $date;
376 # We read two days to find the end_time of the last programme of
377 # the first day by looking at the start_time of the first programme
379 my $enddate = create_dt
( $date, 'UTC' )->add( days
=> 1 )->ymd('-');
381 my( $res, $sth ) = $self->{datastore
}->sa->Sql( "
382 SELECT * from programs
383 WHERE (channel_id = ?)
384 and (start_time >= ?)
386 ORDER BY start_time",
387 [$chd->{id
}, "$startdate 00:00:00", "$enddate 23:59:59"] );
389 my $w = $self->CreateWriter( $chd, $date );
393 my $d1 = $sth->fetchrow_hashref();
395 if( (not defined $d1) or ($d1->{start_time
} gt "$startdate 23:59:59") ) {
396 $self->CloseWriter( $w );
398 EndLogSection
( $section );
402 while( my $d2 = $sth->fetchrow_hashref() )
404 if( (not defined( $d1->{end_time
})) or
405 ($d1->{end_time
} eq "0000-00-00 00:00:00") )
407 # Fill in missing end_time on the previous entry with the start_time
408 # of the current entry
409 $d1->{end_time
} = $d2->{start_time
}
411 elsif( $d1->{end_time
} gt $d2->{start_time
} )
413 # The previous programme ends after the current programme starts.
414 # Adjust the end_time of the previous programme.
415 w
"Adjusted endtime $d1->{end_time} => $d2->{start_time}";
417 $d1->{end_time
} = $d2->{start_time
}
420 $self->WriteEntry( $w, $d1, $chd )
421 unless $d1->{title
} eq "end-of-transmission";
423 if( $d1->{end_time
} lt $d2->{start_time
} )
425 my $parser = DateTime
::Format
::Strptime
->new( pattern
=> '%Y-%m-%d %H:%M:%S' );
426 my $dt1 = $parser->parse_datetime( $d1->{end_time
} );
427 my $dt2 = $parser->parse_datetime( $d2->{start_time
} );
429 my $dur = $dt2 - $dt1;
430 if( defined $self->{copyright
}
431 and $chd->{allowcredits
}
432 and $dur->delta_minutes > $self->{conf
}->{Site
}->{CreditsDuration
}
433 and $self->{conf
}->{Site
}->{AppendCopyright
} eq 1 ){
436 if( $self->{conf
}->{Site
}->{FillHole
} eq 0 ){
437 $copend = $dt1->clone()->add( minutes
=> $self->{conf
}->{Site
}->{CreditsDuration
} );
443 start_time
=> $d1->{end_time
},
444 end_time
=> $copend->ymd() . " " . $copend->hms(),
445 title
=> $self->{copyright
}->{title
},
446 description
=> $self->{copyright
}->{description
},
449 $self->WriteEntry( $w, $dcop, $chd );
451 w
"Copyright appended at $dcop->{start_time} - $dcop->{end_time}";
456 if( $d2->{start_time
} gt "$startdate 23:59:59" ) {
465 # The loop exited because we ran out of data. This means that
466 # there is no data for the day after the day that we
467 # wanted to export. Make sure that we write out the last entry
468 # if we know the end-time for it.
469 if( (defined( $d1->{end_time
})) and
470 ($d1->{end_time
} ne "0000-00-00 00:00:00") )
472 $self->WriteEntry( $w, $d1, $chd )
473 unless $d1->{title
} eq "end-of-transmission";
477 w
"Missing end-time for last entry"
478 unless $date gt $self->{LastRequiredDate
};
482 $self->CloseWriter( $w );
485 EndLogSection
( $section );
491 my( $chd, $date ) = @_;
493 my $xmltvid = $chd->{xmltvid
};
495 my $path = $self->{Root
};
496 my $filename = $xmltvid . "_" . $date . ".xml";
498 $self->{writer_filename
} = $filename;
499 $self->{writer_entries
} = 0;
500 # Make sure that writer_entries is always true if we don't require data
502 $self->{writer_entries
} = "0 but true"
503 if( ($date gt $self->{LastRequiredDate
}) or $chd->{empty_ok
} );
505 open( my $fh, '>:encoding(' . $self->{Encoding
} . ')', "$path$filename.new")
506 or die( "Xmltv: cannot write to $path$filename.new" );
508 my $w = new XMLTV
::Writer
( encoding
=> $self->{Encoding
},
511 $w->start({ 'generator-info-name' => 'nonametv' });
521 my $path = $self->{Root
};
522 my $filename = $self->{writer_filename
};
523 delete $self->{writer_filename
};
527 if( $self->{KeepXml
} ){
528 system("gzip -c -f -n $path$filename.new > $path$filename.new.gz");
529 move
( "$path$filename.new" , "$path$filename" );
531 system("gzip -f -n $path$filename.new");
534 if( -f
"$path$filename.gz" )
536 system("diff $path$filename.new.gz $path$filename.gz > /dev/null");
539 move
( "$path$filename.new.gz", "$path$filename.gz" );
541 if( $self->{KeepXml
} ){
542 move
( "$path$filename.new" , "$path$filename" );
544 if( not $self->{writer_entries
} )
546 w
"Created empty file";
548 elsif( $self->{writer_entries
} > 0 )
550 my @errors = ValidateFile
( "$path$filename.gz" );
551 if( scalar( @errors ) > 0 )
553 w
"Validation failed: " . join( ", ", @errors );
559 unlink( "$path$filename.new.gz" );
560 if( $self->{KeepXml
} ){
561 unlink( "$path$filename.new" );
567 move
( "$path$filename.new.gz", "$path$filename.gz" );
569 if( not $self->{writer_entries
} )
573 elsif( $self->{writer_entries
} > 0 )
575 my @errors = ValidateFile
( "$path$filename.gz" );
576 if( scalar( @errors ) > 0 )
578 w
"Validation failed: " . join( ", ", @errors );
587 my( $w, $data, $chd ) = @_;
589 $self->{writer_entries
}++;
591 my $start_time = create_dt
( $data->{start_time
}, "UTC" );
592 $start_time->set_time_zone( "Europe/Stockholm" );
594 my $end_time = create_dt
( $data->{end_time
}, "UTC" );
595 $end_time->set_time_zone( "Europe/Stockholm" );
598 channel
=> $chd->{xmltvid
},
599 start
=> $start_time->strftime( "%Y%m%d%H%M%S %z" ),
600 stop
=> $end_time->strftime( "%Y%m%d%H%M%S %z" ),
601 title
=> [ [ $data->{title
}, $chd->{sched_lang
} ] ],
604 $d->{desc
} = [[ $data->{description
},$chd->{sched_lang
} ]]
605 if defined( $data->{description
} ) and $data->{description
} ne "";
607 $d->{'sub-title'} = [[ $data->{subtitle
}, $chd->{sched_lang
} ]]
608 if defined( $data->{subtitle
} ) and $data->{subtitle
} ne "";
610 if( defined( $data->{episode
} ) and ($data->{episode
} =~ /\S/) )
612 my( $season, $ep, $part );
614 if( $data->{episode
} =~ /\./ )
616 ( $season, $ep, $part ) = split( /\s*\.\s*/, $data->{episode
} );
617 if( $season =~ /\S/ )
624 w
"Simple episode '$data->{episode}'";
625 $ep = $data->{episode
};
629 my( $ep_nr, $ep_max ) = split( "/", $ep );
632 my $ep_text = $self->{lngstr
}->{episode_number
} . " $ep_nr";
633 $ep_text .= " " . $self->{lngstr
}->{of
} . " $ep_max"
635 $ep_text .= " " . $self->{lngstr
}->{episode_season
} . " $season"
638 $d->{'episode-num'} = [[ norm
($data->{episode
}), 'xmltv_ns' ],
639 [ $ep_text, 'onscreen'] ];
642 # This episode is only a segment and not a real episode.
644 $d->{'episode-num'} = [[ norm
($data->{episode
}), 'xmltv_ns' ]];
648 if( defined( $data->{program_type
} ) and ($data->{program_type
} =~ /\S/) )
650 push @
{$d->{category
}}, [$data->{program_type
}, 'en'];
652 elsif( defined( $chd->{def_pty
} ) and ($chd->{def_pty
} =~ /\S/) )
654 push @
{$d->{category
}}, [$chd->{def_pty
}, 'en'];
657 if( defined( $data->{category
} ) and ($data->{category
} =~ /\S/) )
659 push @
{$d->{category
}}, [$data->{category
}, 'en'];
661 elsif( defined( $chd->{def_cat
} ) and ($chd->{def_cat
} =~ /\S/) )
663 push @
{$d->{category
}}, [$chd->{def_cat
}, 'en'];
666 if( defined( $data->{production_date
} ) and
667 ($data->{production_date
} =~ /\S/) )
669 $d->{date
} = substr( $data->{production_date
}, 0, 4 );
672 if( defined( $data->{aspect
} ) and $data->{aspect
} ne "unknown" )
674 $d->{video
} = { aspect
=> $data->{aspect
} };
677 if( $data->{quality
} )
679 $d->{video
} = { quality
=> $data->{quality
} };
682 if( defined( $data->{stereo
} ) and $data->{stereo
} =~ /\S/ )
684 $d->{audio
} = { stereo
=> $data->{stereo
} };
687 if( defined( $data->{rating
} ) and $data->{rating
} =~ /\S/ )
689 # the 'MPAA' string should not be hardcoded like it is now
690 # it is different for each channel/programmer
691 push @
{$d->{rating
}}, [$data->{rating
}, 'MPAA'];
694 if( defined( $data->{directors
} ) and $data->{directors
} =~ /\S/ )
696 $d->{credits
}->{director
} = [split( ", ", $data->{directors
})];
699 if( defined( $data->{actors
} ) and $data->{actors
} =~ /\S/ )
701 $d->{credits
}->{actor
} = [split( ", ", $data->{actors
})];
702 foreach my $actor (@
{$d->{credits
}->{actor
}} ) {
703 w
"Bad actor $data->{actors}"
704 if( $actor =~ /^\s*$/ );
708 if( defined( $data->{writers
} ) and $data->{writers
} =~ /\S/ )
710 $d->{credits
}->{writer
} = [split( ", ", $data->{writers
})];
713 if( defined( $data->{adapters
} ) and $data->{adapters
} =~ /\S/ )
715 $d->{credits
}->{adapter
} = [split( ", ", $data->{adapters
})];
718 if( defined( $data->{producers
} ) and $data->{producers
} =~ /\S/ )
720 $d->{credits
}->{producer
} = [split( ", ", $data->{producers
})];
723 if( defined( $data->{presenters
} ) and $data->{presenters
} =~ /\S/ )
725 $d->{credits
}->{presenter
} = [split( ", ", $data->{presenters
})];
728 if( defined( $data->{commentators
} ) and $data->{commentators
} =~ /\S/ )
730 $d->{credits
}->{commentator
} = [split( ", ", $data->{commentators
})];
733 if( defined( $data->{guests
} ) and $data->{guests
} =~ /\S/ )
735 $d->{credits
}->{guest
} = [split( ", ", $data->{guests
})];
738 $w->write_programme( $d );
742 # Write description of all channels to channels.xml.gz.
744 sub ExportChannelList
747 my( $channelgroup ) = @_;
749 my $ds = $self->{datastore
};
753 my $odoc = XML
::LibXML
::Document
->new( "1.0", $self->{Encoding
} );
754 my $root = $odoc->createElement('tv');
755 $root->setAttribute( 'generator-info-name', 'nonametv' );
756 $odoc->setDocumentElement($root);
758 my $query = "SELECT * from channels WHERE export=1 ";
761 $query .= "AND chgroup=\'$channelgroup\' ";
763 $query .= "ORDER BY display_name";
764 my( $res, $sth ) = $ds->sa->Sql( $query );
766 while( my $data = $sth->fetchrow_hashref() )
768 my $ch = $odoc->createElement( 'channel' );
769 $ch->setAttribute( id
=> $data->{xmltvid
} );
770 my $dn = $odoc->createElement( 'display-name' );
771 # FIXME why not $data->{sched_lang}?
772 $dn->setAttribute( 'lang', $self->{Language
} );
773 $dn->appendText( $data->{display_name
} );
774 $ch->appendChild( $dn );
776 my $bu = $odoc->createElement( 'base-url' );
777 $bu->appendText( $self->{RootUrl
} );
778 $ch->appendChild( $bu );
782 my $logo = $odoc->createElement( 'icon' );
783 $logo->setAttribute( 'src', $self->{IconRootUrl
} .
784 $data->{xmltvid
} . ".png" );
785 $ch->appendChild( $logo );
788 $root->appendChild( $ch );
794 $outfile = "$self->{Root}channels-$channelgroup.xml";
798 $outfile = "$self->{Root}channels.xml";
800 open( my $fh, '>', $outfile )
801 or die( "Xmltv: cannot write to $outfile" );
804 $odoc->toFH( $fh, 1 );
807 if( $self->{KeepXml
} ){
808 system("gzip -c -f -n $outfile > $outfile.gz");
810 system("gzip -f -n $outfile");
815 # Remove old xml-files and xml.gz-files.
821 my $ds = $self->{datastore
};
823 # Keep files for the last PastDays. (default: one week)
824 my $keep_date = DateTime
->today->subtract( days
=> $self->{PastDays
} )->ymd("-");
826 my @files = glob( $self->{Root
} . "*" );
829 foreach my $file (@files)
832 ($file =~ /(\d\d\d\d-\d\d-\d\d)\.xml(\.gz){0,1}/);
834 if( defined( $date ) )
836 # Compare date-strings.
837 if( $date lt $keep_date )
845 p
"Removed $removed files"
851 ### Setup coding system