#!/usr/bin/perl
#-----------------------------------------------------------------------------
#
#  Tirex Tile Rendering System
#
#  tirex-backend-manager
#
#-----------------------------------------------------------------------------
#  See end of this file for documentation.
#-----------------------------------------------------------------------------
#
#  Copyright (C) 2010  Frederik Ramm <frederik.ramm@geofabrik.de> and
#                      Jochen Topf <jochen.topf@geofabrik.de>
#  
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#  
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#  
#  You should have received a copy of the GNU General Public License
#  along with this program; If not, see <http://www.gnu.org/licenses/>.
#
#-----------------------------------------------------------------------------

use strict;
use warnings;

use Fcntl;
use Getopt::Long qw( :config gnu_getopt );
use IO::Pipe;
use IO::Select;
use IO::Socket;
use POSIX qw(sys_wait_h setsid);
use Pod::Usage qw();
use Socket;
use Sys::Syslog;

use Tirex;
use Tirex::Renderer;
use Tirex::Map;

#-----------------------------------------------------------------------------

die("refusing to run as root\n") if ($< == 0);

#-----------------------------------------------------------------------------
# Reading command line and config
#-----------------------------------------------------------------------------

my @argv = @ARGV;

my %opts = ();
GetOptions( \%opts, 'help|h', 'debug|d', 'foreground|f', 'config|c=s' ) or exit(2);

if ($opts{'help'})
{
    Pod::Usage::pod2usage(
        -verbose => 1,
        -msg     => "tirex-backend-manager - tirex backend manager\n",
        -exitval => 0
    );
}

$Tirex::DEBUG = 1 if ($opts{'debug'});
$Tirex::FOREGROUND = 1 if ($opts{'foreground'});
# debug implies foreground
$Tirex::FOREGROUND =1 if ($opts{'debug'});

my $config_dir = $opts{'config'} || $Tirex::TIREX_CONFIGDIR;
my $config_file = $config_dir . '/' . $Tirex::TIREX_CONFIGFILENAME;
Tirex::Config::init($config_file);

#-----------------------------------------------------------------------------
# Initialize logging
#-----------------------------------------------------------------------------

openlog('tirex-backend-manager', $Tirex::DEBUG ? 'pid|perror' : 'pid',
    Tirex::Config::get('backend_manager_syslog_facility', $Tirex::BACKEND_MANAGER_SYSLOG_FACILITY, qr{^(daemon|syslog|user|local[0-7])$}));

syslog('info', 'tirex-backend-manager started with cmd line options: %s', join(' ', @argv));

Tirex::Config::dump_to_syslog();

#-----------------------------------------------------------------------------
# Daemonize unless in debug mode or foreground
#-----------------------------------------------------------------------------

if (!$Tirex::FOREGROUND)
{
    chdir('/')                     or die("Cannot chdir to /: $!");
    open(STDIN,  '<', '/dev/null') or die("Cannot read from /dev/null: $!");
    open(STDOUT, '>', '/dev/null') or die("Cannot write to /dev/null: $!");
    defined(my $pid = fork)        or die("Cannot fork: $!");
    exit(0) if ($pid);
    setsid()                       or die("Cannot start a new session: $!");
    open(STDERR, '>&STDOUT')       or die("Cannot dup stdout: $!");
}

#-----------------------------------------------------------------------------
# Write pid to pidfile
#-----------------------------------------------------------------------------
my $pidfile = Tirex::Config::get('backend_manager_pidfile', $Tirex::BACKEND_MANAGER_PIDFILE);

if (open(my $pidfh, '>', $pidfile)) 
{
    print $pidfh "$$\n";
    close($pidfh);
}
else
{
    syslog('err', "Can't open pidfile '$pidfile' for writing: $!\n");
    # keep going, we'd rather go on without a pidfile than stopping the show
}

#-----------------------------------------------------------------------------
# Set signal handler
#-----------------------------------------------------------------------------

my $received_sighup  = 0;
my $received_sigterm = 0;

$SIG{'HUP'}  = \&sighup_handler;
$SIG{'TERM'} = \&sigterm_handler;
$SIG{'INT'}  = \&sigterm_handler;

#-----------------------------------------------------------------------------

# workers must check in every so often; note they will not check
# in during tile rendering so allow ample time.
my $ALIVE_TIMEOUT = Tirex::Config::get('backend_manager_alive_timeout', $Tirex::BACKEND_MANAGER_ALIVE_TIMEOUT) * 60; # minutes -> seconds

# a child that has been sent a signal must react this quickly
my $HANGUP_TIMEOUT = 15;
my $TERM_TIMEOUT   =  5;

# timeout when waiting for alive messages from workers
my $SELECT_TIMEOUT = 10;

#-----------------------------------------------------------------------------
# Main loop
#-----------------------------------------------------------------------------
{
    my $sockets = {};
    while (1)
    {
        my $workers = {};

        Tirex::Renderer->read_config_dir($config_dir);
        syslog('info', 'Found config for renderers: %s', join(' ', map { $_->get_name(); } Tirex::Renderer->all()));

        $sockets = open_sockets($sockets);

        $received_sighup = 0;
        $SIG{'HUP'}  = \&sighup_handler;

        while (! $received_sighup)
        {
            start_workers($workers, $sockets);
            check_for_alive_messages($workers);
            cleanup_dead_workers($workers);
            kill_old_workers($workers);

            if ($received_sigterm)
            {
                syslog('info', 'TERM/INT received, forwarding to children: %s', join(' ', keys %$workers));

                foreach my $pid (keys %$workers)
                {
                    kill('HUP', $pid);
                }

                unlink($pidfile); # ignore return code
                exit(0);
            }

            sleep 1;
        }

        syslog('info', 'HUP received, forwarding to children: %s', join(' ', keys %$workers));
        foreach my $pid (keys %$workers)
        {
            kill('HUP', $pid);
        }

        Tirex::Renderer->clear();
        Tirex::Map->clear();
    }
}


#-----------------------------------------------------------------------------
# Open sockets for all renderers or re-use old ones after a SIGHUP
#-----------------------------------------------------------------------------
sub open_sockets
{
    my $old_sockets = shift;

    my $new_sockets = {};

    # go through all renderers and open sockets if they are not already open
    foreach my $renderer (Tirex::Renderer->all())
    {
        my $port = $renderer->get_port();
        if ($old_sockets->{$port})
        {
            syslog('debug', 're-using socket for port %d', $port);
            $new_sockets->{$port} = $old_sockets->{$port};
            delete $old_sockets->{$port};
        }
        else
        {
            my $socket = IO::Socket::INET->new(
                LocalAddr => 'localhost', 
                LocalPort => $port, 
                Proto     => 'udp', 
                ReuseAddr => 1,
            );

            if ($socket)
            {
                syslog('debug', "opened port %d for renderer '%s'", $port, $renderer->get_name());
                $socket->fcntl(Fcntl::F_SETFD, 0); # unset close-on-exec
                $new_sockets->{$port} = $socket;
            }
            else
            {
                syslog('err', "could not open socket on port %d for renderer '%s', renderer disabled", $port, $renderer->get_name());
                $renderer->disable();
            }
        }
    }

    # close all sockets that are not needed by any renderer
    foreach my $port (keys %$old_sockets)
    {
        syslog('info', 'closing socket for port %d', $port);
        $old_sockets->{$port}->close();
    }

    return $new_sockets;
}


#-----------------------------------------------------------------------------
# Start workers if there are less than configured
#-----------------------------------------------------------------------------
sub start_workers
{
    my $workers = shift;
    my $sockets = shift;

    foreach my $renderer (Tirex::Renderer->enabled())
    {
        my $socket = $sockets->{$renderer->get_port()};
        next unless ($socket);

        while ($renderer->num_workers() < $renderer->get_procs())
        {
            my $pipe = create_pipe();

            my $pid = fork();
            if ($pid == 0) # child
            {
                # remove hash with workers because it contains references to open pipes and sockets
                # that should be closed in the child
                $workers = undef;

                $pipe->writer();

                execute_renderer($renderer, $pipe->fileno(), $socket->fileno());

                # if we are here the execute failed
                syslog('err', "Cannot execute renderer %s (%s)", $renderer->get_name(), $renderer->get_path());
                exit($Tirex::EXIT_CODE_DISABLE);
            }
            elsif ($pid > 0) # parent
            {
                $pipe->reader();

                syslog('info', 'renderer %s started with pid %d', $renderer->get_name(), $pid);

                $workers->{$pid} = { 
                    pid             => $pid,
                    last_seen_alive => time(),
                    handle          => $pipe,
                    renderer        => $renderer,
                };
                $renderer->add_worker($pid);
            }
            else
            {
                syslog('crit', 'error in fork(): %s', $!);
            }
        }
    }
}


#-----------------------------------------------------------------------------
# Create pipe for alive message from worker child to parent
#-----------------------------------------------------------------------------
sub create_pipe
{
    my $reader = IO::Pipe::End->new();
    my $writer = IO::Pipe::End->new();

    my $pipe = IO::Pipe->new($reader, $writer);

    $reader->fcntl(F_SETFD, 0); # unset close-on-exec
    $writer->fcntl(F_SETFD, 0); # unset close-on-exec
    $reader->blocking(0);
    $writer->blocking(0);

    return $pipe;
}


#-----------------------------------------------------------------------------
# Check for alive messages from workers
# will return after $SELECT_TIMEOUT seconds, when a message arrived or when
# we caught a signal
#-----------------------------------------------------------------------------
sub check_for_alive_messages
{
    my $workers = shift;

    my $select = IO::Select->new();
    foreach my $worker (values %$workers)
    {
        $select->add([$worker->{'handle'}, $worker]);
    }

    foreach my $handle_wrapper ($select->can_read($SELECT_TIMEOUT))
    {
        my ($handle, $worker) = @$handle_wrapper;
        my $buf;
        $handle->read($buf, 9999);
        if (length($buf) > 0)
        {
            $worker->{'last_seen_alive'} = time();
        }
    }
}


#-----------------------------------------------------------------------------
# Cleanup dead workers
#-----------------------------------------------------------------------------
sub cleanup_dead_workers
{
    my $workers = shift;

    while ((my $pid = waitpid(-1, WNOHANG)) > 0) 
    {
        my $exit_code = $? >> 8;
        my $signal    = $? & 127;
        syslog('warning', 'child %d terminated (exit_code=%d, signal=%d)', $pid, $exit_code, $signal);

        # this will happen if a worker child dies which we don't know about
        # because we got a SIGHUP in between
        next unless ($workers->{$pid});

        # if the return code of the child is something other than $EXIT_CODE_RESTART, the renderer is disabled
        # this does not happen if the worker child was killed because of a timeout
        #if ($exit_code != $Tirex::EXIT_CODE_RESTART && ! defined $workers->{$pid}->{'hungup'})
        #{
        #    my $renderer = $workers->{$pid}->{'renderer'};
        #    if ($renderer->get_debug())
        #    {
        #        syslog('debug', 'renderer %s not disabled because we are in debug mode', $renderer->get_name());
        #    }
        #    else
        #    {
        #        $renderer->disable();
        #        syslog('err', 'disabled renderer %s', $renderer->get_name());
        #    }
        #}

        $workers->{$pid}->{'handle'}->close();
        $workers->{$pid}->{'renderer'}->remove_worker($pid);
        delete $workers->{$pid};
    }
}


#-----------------------------------------------------------------------------
# Kill workers that haven't send an alive message for a while
#-----------------------------------------------------------------------------
sub kill_old_workers
{
    my $workers = shift;

    my $now = time();

    foreach my $worker (values %$workers) 
    { 
        if ($worker->{'last_seen_alive'} < $now - $ALIVE_TIMEOUT)
        {
            if (! defined($worker->{'killed'}) && defined($worker->{'terminated'}) && ($worker->{'terminated'} < $now - $TERM_TIMEOUT))
            {
                syslog('info', "sending KILL to worker '%s' with pid %d (due to timeout)", $worker->{'renderer'}->get_name(), $worker->{'pid'});
                kill('KILL', $worker->{'pid'});
                $worker->{'killed'} = $now;
            }
            elsif (! defined($worker->{'terminated'}) && defined($worker->{'hungup'}) && ($worker->{'hungup'} < $now - $HANGUP_TIMEOUT))
            {
                syslog('info', "sending TERM to worker '%s' with pid %d (due to timeout)", $worker->{'renderer'}->get_name(), $worker->{'pid'});
                kill('TERM', $worker->{'pid'});
                $worker->{'terminated'} = $now;
            }
            elsif (! defined($worker->{'hungup'}))
            {
                syslog('info', "sending HUP to worker '%s' with pid %d (due to timeout)", $worker->{'renderer'}->get_name(), $worker->{'pid'});
                kill('HUP', $worker->{'pid'});
                $worker->{'hungup'} = $now;
            }
        }
    }
}


#-----------------------------------------------------------------------------
# Execute the worker child with the right environment
#-----------------------------------------------------------------------------
sub execute_renderer
{
    my $renderer      = shift;
    my $pipe_fileno   = shift;
    my $socket_fileno = shift;

    $ENV{'TIREX_BACKEND_NAME'}            = $renderer->get_name();
    $ENV{'TIREX_BACKEND_PORT'}            = $renderer->get_port();
    $ENV{'TIREX_BACKEND_SYSLOG_FACILITY'} = $renderer->get_syslog_facility();
    $ENV{'TIREX_BACKEND_MAP_CONFIGS'}     = join(' ', map { $_->get_filename() } $renderer->get_maps());
    $ENV{'TIREX_BACKEND_ALIVE_TIMEOUT'}   = $ALIVE_TIMEOUT - 20; # give the child 20 seconds less than what the parent uses as timeout to be on the safe side
    $ENV{'TIREX_BACKEND_PIPE_FILENO'}     = $pipe_fileno;
    $ENV{'TIREX_BACKEND_SOCKET_FILENO'}   = $socket_fileno;
    $ENV{'TIREX_BACKEND_DEBUG'}           = 1 if ($Tirex::DEBUG || $renderer->get_debug());

    my $cfg = $renderer->get_config();
    foreach my $key (keys %$cfg)
    {
        $ENV{"TIREX_BACKEND_CFG_$key"} = $cfg->{$key};
    }

    exec($renderer->get_path());
}


#-----------------------------------------------------------------------------
# Signal handlers
#-----------------------------------------------------------------------------

sub sighup_handler 
{
    $received_sighup = 1;
    $SIG{'HUP'} = 'IGNORE';
}

sub sigterm_handler 
{
    $received_sigterm = 1;
}


__END__

=head1 NAME

tirex-backend-manager - manages Tirex rendering backend workers

=head1 SYNOPSIS

tirex-backend-manager [OPTIONS] 

=head1 OPTIONS

=over 4

=item B<--help>

Display help message.

=item B<-d>, B<--debug>

Run in debug mode, and pass on the debug flag to workers

=item B<-f>, B<--foreground>

Run in foreground. E.g. when started from systemd service

=item B<--config=DIR>

Use the DIR config directory instead of /etc/tirex.

=back

=head1 DESCRIPTION

The backend manager starts and monitors as many backend workers as given in
the configuration. The backend processes do the actual rendering of tiles.

The backend manager expects each worker process to send an "alive" message in
regular intervals, and will kill the process if it does not do so in time.

The backend manager does not handle render requests in any way; these are read
directly from a local UDP socket by the individual backend processes.

If the backend manager receives a HUP signal, it will relay this signal
to all backends, causing them to exit after completing their current request.
It will reload the renderer and map configuration and re-start all workers
with the new configuration.

=head1 FILES

=over 4

=item F</etc/tirex/tirex.conf>

The configuration file.

=item F</etc/tirex/renderer/*.conf>

Configuration files for renderers

=item F</etc/tirex/renderer/*/*.conf>

Map configurations

=back

=head1 DIAGNOSTICS

The backend manager logs to the I<daemon> syslog facility unless configured
otherwise. In debug mode, logging is also copied to stderr.

=head1 SEE ALSO

L<http://wiki.openstreetmap.org/wiki/Tirex>

=head1 AUTHORS

Frederik Ramm <frederik.ramm@geofabrik.de>, Jochen Topf
<jochen.topf@geofabrik.de> and possibly others.

=cut

#-- THE END ------------------------------------------------------------------
