#! /usr/bin/perl

use strict ;
use warnings ;

use Dmon ; OBB -> import ; Util -> import ;

my @COMMAND_LINE = ( $0, @ARGV ) ;
my @COMMAND_ARGS = qw(stop start reload state) ;

my $PAT_ARGS = join '|', @COMMAND_ARGS ;

my $prog = substr $0, rindex ( $0, '/' ) + 1 ;
my $Usage = <<USAGE ;
Usage: $prog [OPTIONS] [COMMANDS]
OPTIONS  : [-s] [-q] [-v] [-d] [-h] [-i] [-t] [-c file]
COMMANDS : $PAT_ARGS
option s : be silent
option q : be quiet ; only errors
option v : be verbose ; show all actions
option d : show debug info ; internals
option T : trace ; show all debug info
option h : show help ; exit
option t : test config and work ; exit
option i : interactive ; don't fork the daemon
option c : use config <file> ; default [@{[join ',', Conf::FILES]}]
USAGE
sub Usage { die "$_[0]$Usage" ; }
sub Error { die "[error] $prog: $_[0]\n" ; }
sub Warn  { warn "[warn] $prog: $_[0]\n" ; }

# usage: &GetOptions(ARG,ARG,..) defines $opt_ID as 1 or user spec'ed value
# usage: &GetOptions(\%opt,ARG,ARG,..) defines $opt{ID} as 1 or user value
# ARG = 'ID' | 'ID=SPC' | 'ID:SPC' for no-arg, required-arg or optional-arg
# ID  = perl identifier
# SPC = i|f|s for integer, fixedpoint real or string argument

use Getopt::Long ;
Getopt::Long::config ( 'no_ignore_case' ) ;
my %opt = () ; Usage '' unless GetOptions
  ( \%opt, qw(s q v d T h t i c=s version test) ) ;
Usage("Arg count\n") unless @ARGV <= 1 ;
Usage("Bad argument ($ARGV[0]) \n") if @ARGV and $ARGV[0] !~ /^$PAT_ARGS$/ ;

$opt{d} ||= $opt{T} ;
$opt{v} ||= $opt{d} ;
OBB -> Verbosity ( 'Silent'  ) if $opt{s} ;
OBB -> Verbosity ( 'Quiet'   ) if $opt{q} ;
OBB -> Verbosity ( 'Verbose' ) if $opt{v} ;
OBB -> Verbosity ( 'Debug'   ) if $opt{d} ;
OBB -> Verbosity ( 'Trace'   ) if $opt{T} ;
if ( $opt{h} ) { print $Usage ; exit ; }
if ( $opt{version} ) { print Dmon -> Version, "\n" ; exit ; }

my $ARG = shift || '' ;

# printf "%s\n", Util::ddate time ; exit ;

my $conf = Conf -> Make ( file => $opt{c} ) ;

$conf -> PORT ( 23007 ) if $opt{test} ;
$conf -> Dmp ( 'config' ) if $opt{v} ;
Util -> INITx ( conf => $conf ) ;

my $HUP  = 0 ;
my $USR1 = 0 ;
$SIG { INT } = sub { OBB::logx ( 'SIG INT' ) ; exit ; } ;
$SIG { HUP } = sub { OBB::logx ( 'SIG HUP' ) ; $HUP = 1 ; } ;
$SIG { USR1 } = sub { OBB::logx ( 'SIG USR1' ) ; $USR1 = time ; } ;

my $daemon  = Dmon::Daemon -> Make ( conf => $conf )
  -> sss_exit ( $ARG, $opt{i} ) unless $opt{t} ;
my $threads = Threads -> Make unless $opt{t} ;
my $hnam    = $Util::canonical_hostname ;

my $SERVER  = Server -> Make ( conf => $conf, threads => $threads )
  if $conf -> server eq $hnam ;
my $CLIENT  = Client -> Make ( conf => $conf, threads => $threads ) ;
my $itms    = $CLIENT -> work -> hosts -> { $hnam } ;
my $server  = $CLIENT -> server ;
my $stamp   = $CLIENT -> work -> stamp ;
my $srv_hnm = $server -> hnam ;
my $snd_rps = 1 ;

if ( $SERVER )
  { $SERVER -> CLIENT ( $CLIENT ) ;
    $CLIENT -> SERVER ( $SERVER ) ;
  }

sub time_x ($$)
  { my $incr = shift ;
    my $rand = shift ;
    int ( time + $incr - $rand / 2 + rand $rand ) ;
  }

sub reload
  { logq ( 'reload' ) ;
    my $newc ;
    my $eval = eval { $newc = Conf -> Make ( file => $conf -> file ) ; 1 ; } ;
    my $res = 1 ;
    unless ( defined $eval )
      { logq ( "reload bad [$@]" ) ; $res = 0 ; }
    else
      { $conf = $newc ;
        $conf -> PORT ( 23007 ) if $opt{test} ;
        OBB -> Verbosity ( $conf -> loglvl ) ;
        $threads -> stop ;
        $SERVER -> Init ( conf => $conf, threads => $threads ) if $SERVER ;
        $CLIENT -> Init ( conf => $conf, threads => $threads ) ;
        $snd_rps = 1 ;
        logq ( 'reload ok' ) ;
      }
    $res ;
  }

sub mk_next_stat { map { ( $_ => time ) ; } $CLIENT -> state -> hnams ; }

if ( $opt{v} )
  { printf "hostname [%s]\n", $hnam ;
    printf "logdir   [%s]\n", $conf -> logdir ;
    printf "rundir   [%s]\n", $conf -> rundir ;
    printf "rotate   [%s]\n", scalar localtime time + $conf -> rot_ivl ;
    printf "server   [%s]\n", $server ? $srv_hnm : 'no server' ;
    printf "distrib  [%s]\n", $Util::DISTRIB ;
    printf "stamp    [%s]\n", scalar localtime $stamp ;
    printf "levels   [%s]\n", $CLIENT -> show_levels ;
    $CLIENT -> work -> Dmp ( 'client work' ) ;
    if ( $SERVER )
      { printf "ServHist [%s]\n", $SERVER -> CLIENT -> hist ;
        printf "pmaker   [%s]\n", $SERVER -> work -> pmaker ;
        $SERVER -> work -> Dmp ( 'server work' ) ;
      }
  }

printf "my items :\n  [ %s\n  ]\n", join "\n  , ", sort keys %$itms
  if $opt{v} ;

exit if $opt{t} ;

$| = 1 ;

my %next_stat = mk_next_stat ;
my $next_work = time + $conf -> ival_check_work ;
my $next_rotl = time + $conf -> rot_ivl ;
my $next_zaps = time + 30 ;
my $from_server = [] ;
my $work_lm     = undef ;

$CLIENT -> send_server_reboot ( $hnam, $from_server ) ;

while ( 1 )
  { for my $hnam ( $CLIENT -> state -> hnams )
      { if ( time >= $next_stat { $hnam } )
          { my ( $upds, $events ) = $CLIENT -> mk_update ( $hnam ) ;
            if ( $upds )
              { my $hash = $CLIENT -> state -> hist_hash ( $hnam ) ;
                logt ( 'save to history' ) ;
                OBB::Dmp ( $hash, 'hist hash' ) if OBB -> Terse ;
                $CLIENT -> save_hist ( $hnam, $hash ) ;
                OBB::TT 'event %s', $_ for @$events ;
              }
            if ( @$events )
              { logt ( "send events (%s) to $srv_hnm", scalar @$events ) ;
                my $text = Util::as_text ( OBB -> Unbless ( $events ) ) ;
                my $sender = $CLIENT -> send_server
                  ( "EVENTS $text", $from_server ) ;
                logq ( "can't connect to server $srv_hnm" ) unless $sender ;
              }
            $next_stat { $hnam } = time_x $conf -> ival_make_state, 0 ;
          }
      }
    
    if ( time > $next_zaps )
      { my @hnams = $CLIENT -> state -> hnams ;
        logt ( 'do ZAP [%s]', join ',', @hnams ) ;
        $CLIENT -> zap_hist ( $_ ) for @hnams ;
        $CLIENT -> zap_old ( Dmon::T_EVNTS, $conf -> ival_keep_events )
          if $SERVER ;
        $next_zaps = time_x $conf -> zap_ivl, 10 ;
      }

    if ( time >= $CLIENT -> next_send )
      { OBB::TT 'make to server' ;
        logt ( "send REPORT to $srv_hnm" ) ;
        my $text = 'REPORT ' . $CLIENT -> state -> as_text ;
        my $send_server = $CLIENT -> send_server ( $text, $from_server ) ;
        OBB::TT 'make to server done' ;
        my $next = time_x $conf -> ival_send_report, 10 ;
        unless ( $send_server )
          { logq ( "can't connect to server $srv_hnm" ) ;
            $next = time_x 60, 10 ;
          }
        $CLIENT -> next_send ( $next ) ;
        $work_lm = undef ;
      }

    $threads -> do_a_loop ;

    while ( @$from_server )
      { my $recv = shift @$from_server ;
        logq ( "warn [%s]\n", $recv -> resp ) if $recv -> err ;
        logt
          ( $recv -> err
          ? $recv -> resp
          : "received message type [%s]\n", $recv -> type
          ) ;
        logv ( "resp [%s]\n", $recv -> resp ) ;
        logd ( "received:\n%send", $recv -> body ) ;
        if ( ! $recv -> err and $recv -> type eq 'COMMAND REPORT' )
          { $work_lm = $recv -> work_lm ; }
      }

     if ( time > $next_work )
       { my $HUP1 = $CLIENT -> new_work ( $work_lm ) ;
         my $HUP2 = ( $SERVER and $SERVER -> new_work ) ;
         $HUP ||= ( $HUP1 || $HUP2 ) ;
         $next_work = time_x $conf -> ival_check_work, 8 ;
       }
    
     if ( time > $next_rotl )
       { Util::rotate ( $conf ) ; $next_rotl = time + $conf -> rot_ivl ; }
    
    if ( $SERVER and $snd_rps and ! $opt{test} )
      { logt ( "send ALLSEND to $srv_hnm" ) ;
        my $send_server = $CLIENT -> send_server ( 'ALLSEND', $from_server ) ;
        logq ( "can't connect to server $srv_hnm" ) unless $send_server ;
        $snd_rps = 0 ;
      }
  
     if ( $HUP ) { $HUP = 0 ; %next_stat = mk_next_stat if reload ; }
     
     my $rexec = $CLIENT -> rexec || $USR1 ;
     if ( $rexec and time > $rexec )
       { $Util::EXEC = 1 ;
         $Util::STOP = 'rexec' ;
         Dmon::Daemon -> ulock ;
         exit ;
       }
  }

END
  { if ( Util::MODE )
      { syslog
          ( $Util::STOP eq 'dirty'
          ? 'daemon exits dirty'
          : "daemon stopped ($Util::STOP)"
          ) ;
        $conf -> rm_sys_lock ;
      }
    exec @COMMAND_LINE if $Util::EXEC ;
  }
__END__
=pod

=head1 NAME

  dmon - a distributed monitor

=head1 SYNOPSIS

  Usage: dmon [OPTIONS] [COMMANDS]
  OPTIONS  : [-s] [-q] [-v] [-d] [-h] [-i] [-t] [-c file]
  COMMANDS : stop|start|reload|state
  option s : be silent
  option q : be quiet ; only errors
  option v : be verbose ; show all actions
  option d : show debug info ; internals
  option T : trace ; show all debug info
  option h : show help ; exit
  option t : test config and work ; exit
  option i : interactive ; don't fork the daemon
  option c : use config <file> ; default [dmon.conf,/etc/dmon/conf]
  

=head1 DESCRIPTION

Program B<dmon> provides I<system monitoring> on a collection of hosts.
Each host runs C<dmon> as a I<daemon>.

=over 4

=item *

On each host, C<dmon> is a I<client> ;
on one host C<dmon> is also a I<server> ;
and one host is also the (web) I<pagemaker>.

=item *

On each host, C<dmon> monitors I<items> ;
things like I<cpu-load>, I<disk-usage>,
the I<status> of a couple of I<daemons> etc.

=item *

Every minute, C<dmon> computes a fresh value for each item,
and stores these values in a local (sqlite) database.

=item *

Values of items have a I<fitness-level> ; typically :

  fine soso sick critical dead

The I<fitness> of an item is determined by a configurable
(per [host-]item) I<fitness-function>.

=item *

Every 5 minutes C<dmon> sends a I<report> to the server ;
this reports contains current information about the I<items>.

=item *

If/when the fitness of an host-item changes,
C<dmon> sends an I<event> to the server.

On receiving events, the server may send mails
to selected users, depending on the event
(host, item, current level, previous level).

=back

In a few web-pages, B<dmon> shows you :

=over 4

=item *

all B<problems> : host-items that aren't I<fine>

=item *

recent B<events> ; host-item fitness changes

=item *

the B<history> of each host-item ; in a graph ;
per day, hour, day, month or year

=item *

various overviews ; items by host, hosts by item ; for various intervals

=back

I<What> the B<dmon>-system does, is determined by a I<work-config> file.
It contains (among other things) :

=over 4

=item *

a list of hosts and host-groups

=item *

for each host and/or host-group, the items it must monitor

=item *

the designated I<pagemaker>

=item *

fitness-levels and (per host and/or item) fitness-functions

=item *

a list of users and user-groups

=item *

for all fitness-changes, the users that must receive an alert

=back

Program L<mk-dmon-work|http://dmon.science.uu.nl/dist/mk-dmon-work.html>
reads the work-config file and generates a I<work-file> for each client,
the server and the pagemaker.

The system is I<distributed> in the sense that :

=over 4

=item *

each client-dmon runs on its own ; there is no central (work)flow-control

=item *

each client-dmon keeps its own history in a local (sqlite) database

=item *

each client-dmon can talk to the server ; the server can talk to each client ;
all connections are short-lived.

=back

=head1 OPTIONS

=over 4

=item -s

be silent

=item -q

be quiet ; only errors

=item -v

be verbose ; show all actions

=item -d

show debug info ; internals

=item -T

trace ; show all debug info

=item -h

show help ; exit

=item -t

test config and work ; exit

=item -i

interactive ; don't fork the daemon

=item -c

use config I<file> ; default : C<dmon.conf>, C</etc/dmon/conf>

=back

=head1 CONFIG FILE

=head2 location

The default locations of the config file are :

=over 4

=item * B<./dmon.conf>

=item * B</etc/dmon/conf>

=back

=head2 syntax

A config file looks like this :

  +--------------------------------------------------
  |# lines that start with '#' are comment
  |# blank lines are ignored too
  |# tabs are replaced by a space
  |
  |# the config entries are 'key' and 'value' pairs
  |# a 'key' begins in column 1
  |# the 'value' is the rest of the line
  |somekey  part1 part2 part3 ...
  |otherkey part1 part2 part3 ...
  |
  |# indented lines are glued
  |# the next three lines mean 'somekey part1 part2 part3'
  |somekey part1
  |  part2
  |  part3
  +--------------------------------------------------

=head2 config file : required entries

=over 4

=item server I<host>

Specify the (fully qualified) hostname of the server.

=back

=head2 config file : optional entries

=over 4

=item hostname I<host>

=item domain I<domain>

Specify a fully qualified hostname for the client. The default is :

  hostname `hostname`

If C<`hostname`> is not fully qualified (does not contain a dot),
B<dmon> attempts to append a a domain-string.
If defined, B<dmon> uses config-option I<domain> ;
otherwise file C</etc/resolv.conf> is searched for a I<search> list
and the first name found used.

If I<$hostname> resolves to a I<CNAME>, the pointed-to name is used.

Using C<dmon -t -v> tells you which (canonical) I<$hostname> B<dmon> uses.

=item PORT I<port>

Specify a number ; the default is :

  PORT 22007

The server listens for connections on port I<$PORT>.
The client listens for connections on port I<$PORT+1>.

=item ival_make_state I<interval-spec>

Specify the interval after which the client
will compute the next I<state>. The default is (one minute) :

  ival_make_state 1m

An B<interval-spec> can be given in seconds (as in B<22> or B<22s>),
minutes [B<m>],
hours [B<h>],
days [B<d>] and/or
weeks [B<w>].

The I<interval-specs> can be combined in any order :

  dw      # a day and a week
  7d+24h  # same thing
  w-0.5h  # a week minus half an hour
  hm6     # 3666 seconds

=item ival_send_report I<interval-spec>

Specify the interval after which the client
will send the next I<report> to the server. The default is :

  ival_send_report 5m

=item ival_check_work I<interval-spec>

Specify the interval after which the client will request a fresh
I<work-file> from the server. The default is :

  ival_check_work 10m

=item ival_keep_events I<interval-spec>

Specify how long the server must store events.
The default is four weeks :

  ival_keep_events 4w

The server cleans up the event-history hourly.

=item loglvl Silent | Quiet | Terse | Verbose | Debug | Trace

Specify a I<log-level> ; the default is

  loglvl Terse

=item plot_url I<url>

Script L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html> generates references to plotter L<plotter.php|http://dmon.science.uu.nl/dist/plotter.html> as I<$url> ;
the default is :

  plot_url /plotter.php

A leading slash is interpreted as the DOCUMENTROOT of the
vhost running L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html>.

=item rotate I<num> I<interval-spec>

Specify parameters for I<log rotation> ; the default is

  rotate 8 1d

On start-up, and after I<interval-spec> seconds, B<dmon> will
rotate its logfile (C</var/log/dmon/dmon.log>),
saving I<num> files.

=item bindir I<dir>

Specify the directory where the programs live. The default is :

  bindir /usr/sbin

Option I<$bindir> is used by the C<UPGRADE>-facility.
The server sends (and the clients installs in) I<$bindir>C</dmon>.

Option I<$bindir> is ignored by C<root>.

=item logdir I<dir>

Specify the directory where the logfiles live. The default is :

  logdir /var/log/dmon

=item vardir I<dir>

Specify a B<dmon> keeps it files. The default is :

  vardir /var/dmon

=item rundir I<dir>

Specify a I<run-dir>. The default is :

  rundir /var/run/dmon

This is where the daemon keeps its files.

=item lckdir I<dir>

Specify a I<lock-dir>. The default is :

  rundir /var/lock/subsys

=item httpdgid group-name | number

There is no default.
Specify the group (name or number) under which the httpd is running
on the pagemaker.

Gid I<$httpdgid> is used by program L<users-dmon|http://dmon.science.uu.nl/dist/users-dmon.html> to make the user-database
writable for the http-daemon.

=item page_sec I<url>

Specify a (https) I<url> ; there is no default.

I<$page_sec> is used on the I<pagemaker> by cgi-script L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html>
to refer to the secure dmon-web-site, where users can I<login>.

=back

=head1 Client

=head2 Client commands

By default, the clients listens on port 22008 for connections.
The client accepts connections from I<localhost> and the server.
It expects a I<command> from the list below.

To send a command to the client, use I<netcat> (C<nc>) :

  echo <command> | nc localhost 22007
  echo <command> | nc <client-hostname> 22007

=over 4

=item * PING

The server responds :

  COMMAND PING
  PONG from Client <hostname> dmon-0.05-p179
  COMMAND DONE

=item * STATE

The client responds with something like :

  COMMAND STATE
  -- version  dmon-0.05-p184
  -- logfile  /var/log/dmon/dmon.log
  -- loglevel Trace
  -- listening on port 22008 as a Client
  -- Client is processing a command-session << 127.0.0.1 port 22008
  -- hostname science-bs32.science.uu.nl
  -- server   down.science.uu.nl
  -- work     Thu Jan 28 14:21:27 2016
  -- Client state : { ... }
  COMMAND DONE

=item * STOP I<secret>

Command C<STOP> stops the client.
On startup, the daemon generates a I<secret> and stores it (by default)
in file C</var/run/dmon/dmon.stp>, mode C<0600>.
This secret must be supplied in the C<STOP> command.

Since the secret is only available to the owner of the daemon,
only the owner of the daemon can use the C<STOP> command.

=item * HIST [ ival [ pnts [ host [ name ... ] ] ] ]

where

=over 4

=item *

I<ival> represents an interval :
B<H>our, B<D>ay, B<W>eek, B<M>onth or B<Y>ear ;
only the first letter is used ; default B<H>

=item *

I<pnts> is ignored ; default 100

=item *

I<host> is a hostname ; default : the client's canonical hostname

=item *

I<name> is an item-name ; default : all item-names for I<host>

=back

The client queries its history-database, grouping rows
and averaging over groups.

  +----------+------------+------+
  | Interval | Group by   | rows |
  |----------+------------+------|
  | Hour     |  1 minute  |   60 |
  | Day      | 10 minutes |  145 |
  | Week     |  1 hour    |  168 |
  | Month    |  4 hours   |  186 |
  | Year     |  2 days    |  184 |
  +----------+------------+------+

Note ; instead of these constants, we should be able to specify
the approximate number of rows we want (I<$pnts>).
There is a relation with C<ZAP>.

The client responds with a json-encoded summary of the host-item history ;
something like

  COMMAND HIST
  { "resp" : "ok rows 145"
  , "data" : { "cols" : [ "TIME", item1, ... ]
             , "rows" : [ [ time, value, ... ], ... ]
             }
  }
  COMMAND DONE

where I<cols> is a list of (requested) item-names (ordered alphbetically),
and I<rows> is a list of [time,values] tuples.

The C<HIST> command is used by the plotter to retrieve historical data
from a client, so it can generate a plot.

=item * SEND

Command C<SEND> instructs the client to send a report to the server.

The client re-schedules sending its next report to a randomised time
in the near future ; this avoids congestion on the server if/when
all clients receive a C<SEND> command as a result of a server's
C<ALLSEND> command.

The client responds with :

  COMMAND SEND
  next_send down.science.uu.nl Sat Feb 13 18:45:39 2016
  COMMAND DONE

=item * UPGRADE { "text" : program text }

The client expects a json-coded fresh program-text.
It saves and tests the new program and, if ok,
installs the program, sends a reponse, and re-execs itself.

The client responds with something like :

  COMMAND UPGRADE
  upgrade science-vs14.science.uu.nl ... ok dmon-0.05-p183 → dmon-0.05-p184
  ...
  COMMAND DONE

Note: by default (or when running as C<root>), the server sends
(and the client installs in) C</local/sbin/dmon>.

Note: a client accepts an C<UPGRADE> command only from the server.

=item * SERVER I<cmd>

The client connects to the server and issues command I<cmd>.
The clients responds with the server's response.

This facility is used for testing.

=item * REPORT

The client responds with a json-dump of its current state.

This facility is used for testing.

=item * ZAP

The C<ZAP> command instructs the client to I<zap> its history database :
the client reduces the number of rows in its history by replacing
groups of rows by a single row containing the average values of the group.

This facility is used for testing ; clients zaps their history just
after startup, and subsequently every hour.

=back

=head1 Server

=head2 Server commands

By default, the server listens on port 22007 for connections.
The server accepts connections from I<localhost> and the clients.
It expects a I<command> from the list below.

To send a command to the server, use I<netcat> (C<nc>) :

  echo <command> | nc localhost 22007
  echo <command> | nc <server-hostname> 22007

=over 4

=item * PING

The server responds :

  COMMAND PING
  PONG from Server <hostname> <dmon-version>
  COMMAND DONE

=item * STATE

The server responds with something like :

  COMMAND STATE
  -- version  dmon-0.05-p179
  -- logfile  /var/log/dmon/dmon.log
  -- loglevel Trace
  -- listening on port 22008 as a Client
  -- listening on port 22007 as a Server
  -- Server is processing a command-session << 127.0.0.1 port 22007
  -- Server state : keeping state for 74 clients
  COMMAND DONE

=item * WORK I<prog> I<hostname>

where I<prog> == C<dmon-server> | C<dmon-client> | C<dmon-pmaker>

The server responds with something like :

  COMMAND WORK
  { "resp" : "ok work"
  , "data" : contents of the work-file as a string
  , "lm"   : work-file's last-modified timestamp
  }
  COMMAND DONE

where (by default) the work-file is :

  /var/dmon/works/<prog>/<hostname>.txt

On the server, C<dmon> retrieves work-files from the file-system.

The clients and the pagemaker retrieve a work-file from the server ;
except when its (config) server is equal to its (canonical) hostname,
in which case the work-file is retrieved from the file-system.

=item * WORK_LM I<prog> I<hostname>

Same as command C<WORK> except that I<data> is always C<null>.

After startup, periodicly (default every 10 minutes), a client
issues a I<WORK_LM> command to the server, which responds with
the current last-modified timestamp of the client's work-file.

If the timestamp has changed, the client reloads itself.

=item * REPORT I<json-hash>

The server expects a I<client report> : a one-line json-hash
describing the current I<state> of the hosts the client monitors
(a client can monitor other (dumb) hosts like upses, using SNMP).

The json-hash contains the I<state> of one or more hosts :

  { host1 :
      { item1 : ...
      , ...
      }
  , host2 : ...
  }

where per item it has a I<value>, I<fitness>, I<probe-errors> etc.

The server just timestamps and stores the hash ;
it responds with something like :

  COMMAND REPORT
  { "resp": "ok report from <client> [<ip>] for <host1, ...>"
  , "work": <timestamp work-file>
  }
  COMMAND DONE

The I<work>-element is the timestamp of the workfile of the requesting host.
The client caches this info, to avoid a C<WORK_LM> request.

The I<pagemaker> retrieves the combined state of all the hosts
with server-command B<CLIENTS> ; see below.

=item * EVENTS I<json-array>

The server expects a list of I<events>.
Each event describes a change of the fitness of an item on a host ;
attributes : I<hostname>, I<item-name>, I<old fitness-level>,
I<new fitness-level>, I<old value>, I<new value>.

The server stores the events in a database.

The I<pagemaker> may retrieve events from the server using the
server command B<CLIENTS> ; see below.

=item * CLIENTS [HOST] [ITEM] [interval-spec ∥ I<num>]

The server responds with something like :

  COMMAND CLIENTS
  { "resp"   : "ok clients"
  , "cdmp"   : { host : state of host, ... }
  , "events" : { recent events ; optionally selected by time or count }
  }
  COMMAND DONE

If I<HOST> is specified, I<cdmp> only contains the state of I<HOST>.

If I<HOST> and/or I<ITEM> is specified,
I<events> only contains event pertaining to I<HOST> and/or I<ITEM>.

The default for the last argument is I<0> (meaning all events) ;
if the argument looks like a number, only the last I<arg> events are sent ;
otherwise only the events in the specified interval are sent.

The C<CLIENTS> command is issued by the pagemaker when it generates
a html report-page.

=item * CLIENT I<hostname> I<cmd>

The server connects with client I<hostname> and issues command I<cmd>.
The server responds with the client's response.

This facility enables a client-host to talk with another client-host.
It is used by the pagemaker to retrieve history information from clients,
unless the pagemaker runs on the same host as the server
(in which case it can talk directly to any client).

=item * ALLPING

The server issues (in parallel) a C<PING> command to each client.

The server's response is the concatenation of the client-responses.

=item * ALLSEND

The server issues (in parallel) a C<SEND> command to each client.

The server's response is the concatenation of the client-responses ;
something like :

  COMMAND SEND
  next_send down.science.uu.nl Sat Feb 13 18:09:11 2016
  ...
  COMMAND DONE

On the server-host, the client sends an C<ALLSEND> to its server-part,
on start-up and after a I<reload> ; so the server is quickly up-to-date.

=item * UPGRADE

=item * UPGRADE hostname

The server issues a C<UPGRADE> command to each client
(or only client I<hostname>). Sent with the command is
a B<dmon> program text.

The client saves and tests the program text and, if the text is ok,
installs it and then re-execs itself.

The server's response is the concatenation of the client-responses ;
something like :

  COMMAND UPGRADE
  upgrade science-vs14.science.uu.nl ... ok dmon-0.05-p183 → dmon-0.05-p184
  ...
  COMMAND DONE

Note: by default (or when running as C<root>), the server sends
(and the client installs in) C</local/sbin/dmon>.

Note: the server accepts an C<UPGRADE> command only from I<localhost> ;
clients accept an C<UPGRADE> command only from the server.

=back

=head1 INSTALL

=head2 Requirements

=over 4

=item *

Daemon C<dmon> must run as C<root> because many system diagnostic tools
can only be used as C<root>.

=item *

The I<server> must be able to connect to all the clients on port B<22008>.
The server listens on port B<22007> ;
It allows connections from I<localhost> and the clients, rejecting others.

=item *

A I<client> must be able to connect to the server on port B<22007>.
A client listens on port B<22008> ;
it allows connections from I<localhost> and the server, rejecting others.

=item *

Program C<dmon> requires a bunch of perl modules, most of which
are I<CORE> (come with C<perl>) ; the others are widely available
from platform-repo's.

=item *

A C<dmon> system works best if the I<server> runs on I<real> hardware
that is not dependent on other hosts you want to monitor.
This gives you the best chance that monitoring is available when
some calamity occurs.

Whatever your initial setup may be, it is easy to (later) move the
C<dmon>-server to another host ; idem for the required web-service.

=item *

To use C<dmon>, one of the monitored hosts must be a I<web-server> ;
it is efficient (but not necessary) to have the I<server> and
the I<web-service> on the same host.

=back

=head2 Installation

=head3 Installation - server

=over 4

=item * Fetch the software

Create some directory ; for instance C</local/dmon/> ; then get C<dmon> :

  % rsync -avz dmon.science.uu.nl::dmon-dev/ /local/dmon/

=item * Install perl modules :

  % cd /local/dmon/
  % perl dmon -v -t

Perl will probably complain with something like :

  Can't locate xxx.pm ...

... where I<xxx> is a missing modules.

Install the module with your favorite tool (yum, apt-get),
or use program C<cpanm>.

Installing modules with C<cpan(1)> is usually horrible ;
use C<cpanm> instead ; see the C<INSTALL> file for hints ; also CPAN's
L<How to install CPAN modules|http://www.cpan.org/modules/INSTALL.html>.

If C<cpanm> fails, view the C<cpanm>-build-log ;
some modules require C<gcc(1)>.

=item * Install perl modules (repeat)

Install missing perl modules until C<dmon> complains about
a missing I<config file>.

=item * create a dmon-config file in C</etc/dmon/conf>

Supply the server's hostname or (preferably) a CNAME
for your server-host, so you can switch later.

  server ...
  loglvl Verbose

=item * create a dmon-work-config file in C</etc/dmon/work>

  # define a host
  host hostname-of-your-server myhost
  host hostname-of-your-webserver www
  pmaker www
  # set fitness levels
  fit_level fine
  fit_level soso
  fit_level sick
  fit_level crit critical
  fit_level dead
  # on myhost, monitor some items
  get myhost cpu_load root_avail root_usage uptime

... where I<hostname-of-your-server> is a fully-quallified hostname,
and I<myhost> is just a unique tag, used to refer to the host.

=item * create dmon work-files in C</var/dmon/works/>

Run program L<mk-dmon-work|http://dmon.science.uu.nl/dist/mk-dmon-work.html> :

  % mk-dmon-work -v -f

=item * try dmon

Start the daemon ; for now, don't fork (use C<-i>) :

  % dmon start -i
  # stop it with ^C

If all is well, you will see C<dmon> start up, first as server, then as client.

After a while,

=over 4

=item *

every minute : the daemon (as client) computes fresh item values ;

=item *

every five minutes :

=over 4

=item *

the daemon (as client) sends a report to the server ;

=item *

the daemon (as server) acceps the report, responds to the client ;

=item *

the client receives the response from the server ; closes the connection.

=back

=back

=item * install C<dmon> as a I<service>

=over 4

=item *

copy C<init.d> :

  % cp init.d /etc/init.d/dmon

=item *

Make sure C<dmon> is started after a reboot ;
use C<chkconfig> :

  % chkconfig --add dmon

or (on Ubuntu) :

  % update-rc.d dmon defaults 50 50

=back

=back

=head3 Installation - client

=over 4

=item * Fetch the software

Same as server-install.

=item * Install missing perl modules

Same as server-install.

=item * create a dmon-config file in C</etc/dmon/conf>

Supply the server's hostname or (preferably) a CNAME
for your server-host, so you can switch later.

  server ...

Configure C<hostname> if necessary ; see config option C<hostname>.

=item * install C<dmon> as a I<service>

Same as server-install.

=back

=head3 Installation - pagemaker

The I<pagemaker> must be a dmon-host and a web-server.

=over 4

=item * install L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html> on the webserver

=over 4

=item *

Make sure the webserver runs L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html> as a cgi-script.

=item *

Script L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html> generates a login-reference ; preferably to a secure site ;
configure something like :

  page_sec https://dmon.your.org/cgi-bin/gen-dmon-page

Just use I<http> instead of I<https> if you don't have a secure site (yet).

=back

=item * install L<plotter.php|http://dmon.science.uu.nl/dist/plotter.html> on the webserver

=over 4

=item *

Make sure the webserver runs L<plotter.php|http://dmon.science.uu.nl/dist/plotter.html> as a php-script.

=item *

Script L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html> generates references to the plotter as C</plotter.php>.
Configure option C<plot_url> to change that.

=item *

The plotter uses package L<jpgraph|http://jpgraph.net/> ;
install the package and make sure the plotter can do :

  require_once ( 'jpgraph/jpgraph.php' ) ;
  require_once ( 'jpgraph/jpgraph_canvas.php' ) ;
  require_once ( 'jpgraph/jpgraph_line.php' ) ;
  require_once ( 'jpgraph/jpgraph_date.php' ) ;

That is, if your plotter lives in :

  /path/to/plotter.php

install C<jpgraph> in :

  /path/to/jpgraph-<VERSION>/

and make a symlink :

  % ln -s /path/to/jpgraph-<VERSION>/src/ /path/to/jpgraph/

=back

=back

=head1 PRODUCTION

Here are some tips for using L<dmon|http://dmon.science.uu.nl/dist/dmon.html> :

=over 4

=item *

In general, monitoring works best if the server runs on stand-alone hardware,
that doesn't use any other resources.
B<Dmon> is very light-weight, so some old box will probably work.

=item *

Dress up the server as a I<web-server> and
run the pagemaker (L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html> and L<plotter.php|http://dmon.science.uu.nl/dist/plotter.html>) on the server ;
this is a little more efficient.

=item *

To make installing B<dmon> on clients easier,
dress up the server as a I<rsync-server> ; then :

=over 4

=item *

Create a module C<[dmon]> containing the downloaded B<dmon> distro.

=item *

In module C<[dmon]> copy the C<Makefile> that comes with B<dmon>,
to file C<makefile> :

  cp Makefile makefile

Tweak the C<makefile> to meet your local needs.

Note : for C<make>, the C<makefile> takes precedence over C<Makefile>.

=back

On a client, rsync B<dmon> from the rsync server :

  % mkdir -p /local/dmon/
  % rsync -avz dmon.your.org::dmon/ /local/dmon/

Then, on the client you can run a C<make> :

  % cd /local/dmon/
  # repeat until all perl-modules are installed :
  % ./dmon -t -v 
  % make install
  # start dmon
  % /etc/init.d/dmon start
  # fix iptables ; allow tcp connections from dmon.your.org:22008

=back

=head1 SIGNALS

  HUP  : dmon reloads  ; dmon re-reads the config ; re-inits client and server
  USR1 : dmon re-execs ; dmon stops with END { exec $PROGRAM_NAME, @ARGV }

=head1 FILES

=over 4

=item * all hosts

  /etc/dmon/conf         dmon-configuration
  /etc/init.d/dmon       dmon start/stop script
  /var/dmon/data.lite    database for item-history data
  /var/dmon/probes       dmon probes ; installed by dmon on startup
  /var/lock/subsys/dmon  touched on dmon-startup ; removed on stop
  /var/log/dmon          logfiles
  /var/run/dmon/dmon.lck lock-file
  /var/run/dmon/dmon.pid pid-file
  /var/run/dmon/dmon.stp dmon stop-secret ; for client STOP-command


=item * server only

  /etc/dmon/work         work-configuration
  /var/dmon/works        per/client work-files 
  /var/dmon/works.tmp    staging directory for /var/dmon/works

=item * pagemaker only

  /var/dmon/cgi-data     database for gen-dmon-page ; login/out-log
  /var/dmon/cgi-secret   secret for cookie-verification ; should be 0600

=back

=head1 SEE ALSO

L<mk-dmon-work|http://dmon.science.uu.nl/dist/mk-dmon-work.html>, L<gen-dmon-page|http://dmon.science.uu.nl/dist/gen-dmon-page.html>, L<users-dmon|http://dmon.science.uu.nl/dist/users-dmon.html>


=head1 LICENSE

You may distribute under the terms of either the GNU General Public
License or the Artistic License, as specified in the Perl 5.10.0 README
file.

=head1 AUTHOR

=for html dmon &copy; 2015-2016
<a href="http://www.staff.science.uu.nl/~penni101">Henk P. Penning</a>
- All rights reserved
; dmon-0.05-p214 - Thu Oct 13 17:38:37 2016

=for man Henk P. Penning, <http://www.staff.science.uu.nl/~penni101> ;
dmon-0.05-p214 - Thu Oct 13 17:38:37 2016


=cut