ONO::Cron::CronSync::Monitor
package ONO::Cron::CronSync::Monitor;
################################################################################
# COPYRIGHT / LICENSE #
################################################################################
#
# This file is part of the ONO Software Project.
#
# Copyright (C) 2000-2025 Jos KIRPS [ www.kirps.com | jos_AT_kirps_DOT_com ]
# and The Joopita Project [ www.joopita.org | contact_AT_joopita_DOT_com ]
#
# This file, as well as other parts of the ONO Software Project or related
# elements, are FREE SOFTWARE available under the ARTISTIC LICENSE 2.0.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# For the full license, see /ono/osr/license/LICENSE.txt, or write to
# jos_AT_kirps_DOT_com or contact_AT_joopita_DOT_com.
#
################################################################################
# END OF COPYRIGHT / LICENSE, HERE COMES THE CODE ... #
################################################################################
use strict;
use ONO::IO;
use ONO::DB;
use ONO::Render;
use ONO::Cron::CronSync::ToolBox;
use ONO::ToolBox::Logfile;
use ONO::Core::HostOS;
use ONO::Core::HostIO;
use ONO::Lib::DateTime::ToolBox;
use ONO::ToolBox::SendMail;
use ONO::Lib::Code::RandomID;
use ONO::Lib::Lang::BLK;
use ONO::FW::IBS::Network;
###############################################################################
# ONO
###############################################################################
#: Monitoring related services for ONO's CronSync service.
sub run {
my (
$self,
$CRON_DIR,
$job,
$data_ref,
$TYPE_ref,
$vars_ref,
$vpath,
$sec,$min,$hour,
$mday,$mon,$year,
$wday,$yday,
$timestamp,
$switches,
) = @_;
#: Execution logic.
my %data = %$data_ref;
my %TYPE = %$TYPE_ref;
my %vars = %$vars_ref;
my $db = ONO::Cron::CronSync::ToolBox->connect();
my ($DATA,$MAIL);
if ($data{'type'} eq "monitor") {
# create the monitoring dir
if (!ONO::IO->exists("var/log/monitoring/machines")) {
ONO::IO->mkpath("var/log/monitoring/machines");
}
if ($db) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database connected [OK]",$switches);
foreach my $line (ONO::DB->select($db,"ono_network","relation = 'ono' AND status > 0 AND flags LIKE '%m%'")) {
my @col = ONO::DB->readcols($line);
if ($col[26] || $col[27]) {
if (!$col[26]) {
$col[26] = $col[27];
}
my $HTML = ONO::IO->curl($col[26]);
$HTML =~ s~(\n|\r|\t)~~g;
my $LOG = "OFFLINE";
if ($HTML =~ /\<body/i && $HTML =~ /\<\/body\>/i && $HTML =~ /ono\.css/i && $HTML !~ /document has moved/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': $col[26] is ONLINE [OK]",$switches);
$LOG = "ONLINE";
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': $col[26] is OFFLINE [ERROR]",$switches);
}
ONO::ToolBox::Logfile->log("var/log/monitoring/$year$mon$mday.log","cronsync","$col[1]: $LOG [$col[26]]");
}
}
foreach my $line (ONO::DB->select($db,"ono_network","relation = 'ono' AND status > 0 AND flags LIKE '%M%'")) {
my @col = ONO::DB->readcols($line);
my $FILE = ONO::IO->load("var/community/ono/network/devices/$col[1]/syslog_exportfile.txt");
$FILE =~ s~(\n|\r|\t)~~g;
if ($FILE) {
my $HTML = ONO::IO->curl($FILE);
my $STAT = "FAILED";
if ($HTML && $HTML !~ /document has moved/ && $HTML !~ /\<html\>/ && length $HTML < 1024) {
$STAT = "SUCCESS";
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ID '$col[1]' Syslog ExportFile: $HTML",$switches);
if (!ONO::IO->exists("var/log/monitoring/syslog/$col[1]")) {
ONO::IO->mkpath("var/log/monitoring/syslog/$col[1]");
}
ONO::IO->store("var/log/monitoring/syslog/$col[1].txt",$HTML);
$HTML =~ s~;~,~g;
ONO::ToolBox::Logfile->log("var/log/monitoring/syslog/$col[1]/$year$mon$mday.log","cronsync","$col[1]: $HTML");
}
ONO::IO->store("var/log/monitoring/syslog/$col[1].status","CURL $FILE : $STAT");
}
}
my (%units,%units_current,%units_counter,%units_online,%units_offline,%units_5m,%units_1h,%sql);
foreach my $line (reverse ONO::IO->list("var/log/monitoring/$year$mon$mday.log")) {
my @lp = split(/;/,$line);
$lp[5] =~ m~^(.*?): (.*?) ~;
my $unit = $1;
my $status = $2;
$units{$unit} .= "$lp[2]:$status\n";
$units_counter{$unit}++;
if ($status eq "ONLINE") {
$units_online{$unit}++;
}
if ($status eq "OFFLINE") {
$units_offline{$unit}++;
}
if ($lp[2] =~ /^$hour$min/) {
$units_current{$unit} = $status;
}
if ($units_counter{$unit} < 6 && $status eq "ONLINE") {
$units_5m{$unit}++;
}
if ($units_counter{$unit} < 61 && $status eq "ONLINE") {
$units_1h{$unit}++;
}
}
foreach my $unit (keys %units) {
if (!$units_current{$unit}) {
$units_current{$unit} = "OFFLINE";
}
if (!$units_online{$unit}) {
$units_online{$unit} = 0;
}
if (!$units_offline{$unit}) {
$units_offline{$unit} = 0;
}
my ($per_on,$per_off) = (0,0);
my ($per_5m,$per_1h) = (0,0);
if ($units_5m{$unit}) {
$per_5m = $units_5m{$unit};
}
if ($units_1h{$unit}) {
$per_1h = $units_1h{$unit};
}
if ($units_counter{$unit}) {
$per_on = (int(1000/$units_counter{$unit}*$units_online{$unit}))/10;
$per_off = (int(1000/$units_counter{$unit}*$units_offline{$unit}))/10;
}
my $sql_status = "ERROR";
my $DATA = "CURRENT:$units_current{$unit}\nCOUNTER:$units_counter{$unit}\nONLINE:$units_online{$unit}\nOFFLINE:$units_offline{$unit}\n";
$DATA .= "ONLINE_PERCENT:$per_on\nOFFLINE_PERCENT:$per_off\nONLINE_5MIN:$per_5m\nONLINE_1HOUR:$per_1h\nSQL_STATUS:$sql_status\n";
ONO::IO->store(
"var/log/monitoring/machines/$unit.txt",
"$DATA$units{$unit}",
);
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: no database connection [ABORT]",$switches);
}
}
if ($data{'type'} eq "monitor_syslog") {
# system load and memory tests
my $cores = ONO::Core::HostOS->cpu_cores;
my ($load1,$load2,$load3) = ONO::Core::HostOS->cpu_load;
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': CPU: $cores,$load1,$load2,$load3",$switches);
my ($mem,$used) = ONO::Core::HostOS->mem_load;
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': MEM: $mem,$used",$switches);
# test if the database is really running
my $SQL_TEST = "ERROR";
my $sql = 0;
my $ID = ONO::Lib::Code::RandomID->make;
if ($db) {
ONO::DB->command($db,"INSERT INTO ono_test (id_10) VALUES ('$ID');");
foreach my $line (ONO::DB->select($db,"ono_test","id_10 = '$ID'")) {
my @col = ONO::DB->readcols($line);
$SQL_TEST = "OK";
$sql = 1;
}
ONO::DB->command($db,"DELETE FROM ono_test;");
}
# create the syslog
unless (ONO::IO->exists("var/log/syslog")) {
ONO::IO->mkdir("var/log",777);
ONO::IO->mkdir("var/log/syslog",777);
}
ONO::ToolBox::Logfile->log("var/log/syslog/$year$mon$mday.log","syslog","CPU/$cores: $load1,$load2,$load3; MEM: $mem,$used; SQL: $SQL_TEST");
if (ONO::IO->exists("etc/syslog.conf")) {
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': found: /etc/syslog.conf",$switches);
foreach my $line (ONO::IO->list("etc/syslog.conf")) {
if ($line =~ m~^ExportFile (.*)$~) {
my $FILE = $1;
$FILE =~ s~[^A-Za-z0-9\.\-\_\/]~~g;
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': export to file: $FILE",$switches);
ONO::IO->store($FILE,"$timestamp;$year$mon$mday;$hour$min$sec;;syslog;$cores,$load1,$load2,$load3,$mem,$used,$sql;;");
}
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': missing: /etc/syslog.conf",$switches);
}
}
if ($data{'type'} eq "monitor_export") {
my $db = ONO::Cron::CronSync::ToolBox->connect();
if ($db) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database connected [OK]",$switches);
if (!ONO::IO->exists("monitoring/advanced")) {
ONO::IO->mkdir("monitoring");
ONO::IO->mkdir("monitoring/advanced");
}
my ($BLK_ref,$UCBLK_ref) = ONO::Lib::Lang::BLK->get("en",$vars_ref);
ONO::IO->mkpath("monitoring/advanced");
ONO::IO->store("monitoring/index.shtml","<html><head></head><body></body></html>");
ONO::IO->store("monitoring/advanced/index.shtml","<html><head></head><body></body></html>");
my $CSSJS = ONO::Render->cssjs();
my %setup;
$setup{'sql_table'} = "ono_network";
$setup{'sql_col_prefix'} = "net";
$setup{'sql_relfield'} = "relation";
$setup{'sql_relation'} = "ono";
my $HTML = ONO::FW::IBS::Network->monitor(
$db,
"ono",
"/ono/admin/",
"en",
\%setup,
$vars_ref,
$vars_ref,
$BLK_ref,
);
ONO::IO->store(
"monitoring/export.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
$setup{'mini_monitoring'}++;
my $HTML = ONO::FW::IBS::Network->monitor(
$db,
"ono",
"/ono/admin/",
"en",
\%setup,
$vars_ref,
$vars_ref,
$BLK_ref,
);
ONO::IO->store(
"monitoring/export-mini.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
$setup{'mini_monitoring'} = 0;
$setup{'advanced_monitoring'}++;
my $HTML = ONO::FW::IBS::Network->monitor(
$db,
"ono",
"/ono/admin/",
"en",
\%setup,
$vars_ref,
$vars_ref,
$BLK_ref,
);
my $KEY;
if ($data{'export_advanced'}) {
$KEY = "-$data{'export_advanced'}";
}
ONO::IO->store(
"monitoring/advanced/export$KEY.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
ONO::IO->store(
"monitoring/advanced/export$KEY-$year$mon$mday.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: no database connection [ABORT]",$switches);
}
}
if ($data{'type'} eq "monitor_backups") {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': $TYPE{$data{'type'}} ($data{'type'})",$switches);
my $daysecs = 24*60*60;
my $daysecs_plus = 26*60*60; # allow 2 hours to complete a backup
my ($REPORT,$STATUS);
my $STATUS_LED = "red";
foreach my $line (split(/\<br\>/,$data{'mon_config'})) {
if ($line !~ /^\#/) {
$REPORT .= qq~$line<br>~;
}
if ($line =~ s/\#\!//) {
my @lp = split(/:/,$line);
if ($lp[0] eq "backup") {
my $STAT = qq~<span class="lightred bold">not found</span>~;
if (ONO::Core::HostIO->exists("$lp[1]/BACKUP_TIMESTAMP.txt") || ONO::Core::HostIO->exists("$lp[1]/var/BACKUP_TIMESTAMP.txt")) {
my $time = ONO::Core::HostIO->load("$lp[1]/BACKUP_TIMESTAMP.txt");
if (!$time) {
$time = ONO::Core::HostIO->load("$lp[1]/var/BACKUP_TIMESTAMP.txt");
}
$STAT = qq~<span class="lightgreen bold">found</span>, ~;
if (length $time == 10) {
$STAT .= ONO::Lib::DateTime::ToolBox->get($time,0,"iz","en");
}
if ($lp[2] =~ /(d|w|m|y)/) {
my $STATUS_TIME = qq~<span class="lightgreen bold">OK</span>~;
if (
($lp[2] =~ /d/ && $time < ($timestamp - $daysecs_plus)) ||
($lp[2] =~ /D/ && $time < ($timestamp - $daysecs_plus*3)) ||
($lp[2] =~ /w/ && $time < ($timestamp - $daysecs*7)) ||
($lp[2] =~ /m/ && $time < ($timestamp - $daysecs*31)) ||
($lp[2] =~ /y/ && $time < ($timestamp - $daysecs*366))
) {
$STATUS_TIME = qq~<span class="lightred bold">OUTDATED</span>~;
$STATUS = "ERROR";
}
$STAT .= " [$STATUS_TIME]";
}
} else {
$STATUS = "ERROR";
}
$REPORT .= qq~<span class="bold">$lp[3]</span>: $STAT<br><span class="col9">\+ $lp[1]</span><br>~;
if ($lp[2] !~ /Q/) {
my $SQL_FILE = "$lp[1]/var/BACKUP_MYSQLDUMP.sql";
if (!ONO::Core::HostIO->exists($SQL_FILE)) {
foreach my $test (ONO::Core::HostIO->ls($lp[1])) {
if ($test !~ /^\./ && $test =~ /[A-Za-z0-9]/ && ONO::Core::HostIO->exists("$lp[1]/$test/latest/sql")) {
foreach my $sql (ONO::Core::HostIO->ls("$lp[1]/$test/latest/sql")) {
if ($sql !~ /^\./ && $sql =~ /[A-Za-z0-9]/ && $sql =~ /\.sql$/) {
$SQL_FILE = "$lp[1]/$test/latest/sql/$sql";
}
}
}
}
}
if (ONO::Core::HostIO->exists($SQL_FILE)) {
my $size = ONO::Core::HostIO->size($SQL_FILE);
my $time = ONO::Core::HostIO->timestamp($SQL_FILE);
my $unit = "b";
if ($size > 8192) {
$size = 1+int($size/1024);
$unit = "kb";
if ($size > 8192) {
$size = 1+int($size/1024);
$unit = "MB";
}
}
my $STATUS_TIME = qq~<span class="lightred">UNKNOWN</span>~;
if (length $time == 10) {
if (
($lp[2] =~ /d/ && $time < ($timestamp - $daysecs)) ||
($lp[2] =~ /w/ && $time < ($timestamp - $daysecs*7)) ||
($lp[2] =~ /m/ && $time < ($timestamp - $daysecs*31)) ||
($lp[2] =~ /y/ && $time < ($timestamp - $daysecs*366))
) {
$STATUS_TIME = qq~<span class="lightred bold">OUTDATED</span>~;
} else {
$STATUS_TIME = qq~<span class="lightgreen bold">OK</span>~;
}
$time = ONO::Lib::DateTime::ToolBox->get($time,0,"iz","en");
}
$REPORT .= qq~<span class="col9">\+ SQL backup <span class="lightgreen bold">found</span>, <span class="bold">$size</span>$unit, $time [$STATUS_TIME]</span><br>~;
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">no SQL backup found</span></span><br>~;
}
if ($lp[2] =~ /q/) {
if (ONO::Core::HostIO->exists("$lp[1]/etc/sql/ono.conf")) {
my %conf;
foreach my $line (ONO::Core::HostIO->list("$lp[1]/etc/sql/ono.conf")) {
$line =~ s~(\n|\r|\t)~~g;
$line =~ m~^(.*?) (.*?)$~;
$conf{$1} = $2;
}
my $db = ONO::DB->connect($conf{'database'},"d",$conf{'username'},$conf{'password'});
if ($db) {
my $time = ONO::DB->get($db,"value","ono_status","name = 'ono_db_timestamp'");
if (length $time == 10) {
my $STATUS_TIME;
if (
($lp[2] =~ /d/ && $time < ($timestamp - $daysecs)) ||
($lp[2] =~ /w/ && $time < ($timestamp - $daysecs*7)) ||
($lp[2] =~ /m/ && $time < ($timestamp - $daysecs*31)) ||
($lp[2] =~ /y/ && $time < ($timestamp - $daysecs*366))
) {
$STATUS_TIME = qq~<span class="lightred bold">OUTDATED</span>~;
} else {
$STATUS_TIME = qq~<span class="lightgreen bold">OK</span>~;
}
$time = ONO::Lib::DateTime::ToolBox->get($time,0,"iz","en");
$REPORT .= qq~<span class="col9">\+ SQL live backup is <span class="lightgreen bold">ONLINE</span>, updated $time [$STATUS_TIME]</span><br>~;
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">No ono_db_timestamp entry found in $conf{'database'} / ono_status</span></span><br>~;
}
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">Cannot connect to the database ($conf{'database'},$conf{'username'},PWD:XXXXXXXX)</span></span><br>~;
}
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">Missing /etc/sql/ono.conf ($lp[1]/etc/sql/ono.conf)</span></span><br>~;
}
}
}
}
if ($lp[0] eq "disk") {
foreach my $disk (ONO::IO->disks) {
my @dp = split(/\^/,$disk);
if ($lp[1] eq $dp[0] || $lp[1] eq "$dp[0]/") {
for (my $i = 1; $i < 4; $i++) {
$dp[$i] = 1+int($dp[$i]/1024/1024);
}
$dp[4] =~ s~[^0-9]~~g;
my $color = "lightgreen";
if ($dp[4] > 80 && $dp[3] < 250) {
$color = "orange";
}
if ($dp[4] > 90 && $dp[3] < 100) {
$color = "lightred";
}
if ($dp[3] > 49) {
$color = "orange";
}
if ($dp[3] > 99) {
$color = "lightgreen";
}
$REPORT .= qq~$lp[3]: <span class="$color bold">$dp[4]%</span>
(<span class="$color bold">$dp[2]</span>GB of <span class="$color bold">$dp[1]</span>GB used, <span class="$color bold">$dp[3]</span>GB free)<br>
~;
}
}
}
}
}
if (!$STATUS) {
$STATUS = "OK";
$STATUS_LED = "green";
}
$REPORT .= qq~<br><br>Report has been generated on $year/$mon/$mday \@ $hour:$min:$sec~;
my $CSSJS = ONO::Render->cssjs();
my $REPORT_MAIL = $REPORT;
my $REPORT_HTML = $REPORT;
$REPORT_HTML =~ s~\+ ~ \+ ~g;
ONO::IO->mkpath("monitoring/backups");
ONO::IO->store("monitoring/index.shtml","<html><head></head><body></body></html>");
ONO::IO->store("monitoring/backups/index.shtml","<html><head></head><body></body></html>");
my $KEY;
if ($data{'mon_file'} =~ /[A-Za-z0-9]/) {
$KEY = "-$data{'mon_file'}";
}
ONO::IO->store(
"monitoring/backups/export$KEY.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
<div class="p20">
<h1>$data{'mon_title'} \[$STATUS\]</h1>
<div class="mb10">$year/$mon/$mday \@ $hour:$min:$sec</div>
$REPORT
</div>
</body>
</html>
~,
);
ONO::IO->store(
"monitoring/backups/export$KEY-mini.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
<table class="default_table auto">
<tr>
<td><img class="block12 auto" src="/ono/osr/images/leds/16x16/$STATUS_LED.png" alt=""></td>
<td class="small">$data{'mon_title'} \[$STATUS\]</td>
</tr>
</table>
</body>
</html>
~,
);
if ($data{'mon_email_hh'} && $data{'mon_email_hh'} == $hour && $data{'mon_email_mm'} && $data{'mon_email_mm'} == $min) {
$REPORT_MAIL =~ s~\ \;~ ~g;
$REPORT_MAIL =~ s~\<br\>~\n~g;
ONO::ToolBox::SendMail->sendmail(
"mail:noreply",
"$data{'mon_email'}",
"[ONO] $data{'mon_title'} \[$STATUS\]",
"$REPORT_MAIL\n\n",
);
}
}
return ($DATA,$MAIL);
}
###############################################################################
# end of script
###############################################################################
1;
__END__
################################################################################
# COPYRIGHT / LICENSE #
################################################################################
#
# This file is part of the ONO Software Project.
#
# Copyright (C) 2000-2025 Jos KIRPS [ www.kirps.com | jos_AT_kirps_DOT_com ]
# and The Joopita Project [ www.joopita.org | contact_AT_joopita_DOT_com ]
#
# This file, as well as other parts of the ONO Software Project or related
# elements, are FREE SOFTWARE available under the ARTISTIC LICENSE 2.0.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# For the full license, see /ono/osr/license/LICENSE.txt, or write to
# jos_AT_kirps_DOT_com or contact_AT_joopita_DOT_com.
#
################################################################################
# END OF COPYRIGHT / LICENSE, HERE COMES THE CODE ... #
################################################################################
use strict;
use ONO::IO;
use ONO::DB;
use ONO::Render;
use ONO::Cron::CronSync::ToolBox;
use ONO::ToolBox::Logfile;
use ONO::Core::HostOS;
use ONO::Core::HostIO;
use ONO::Lib::DateTime::ToolBox;
use ONO::ToolBox::SendMail;
use ONO::Lib::Code::RandomID;
use ONO::Lib::Lang::BLK;
use ONO::FW::IBS::Network;
###############################################################################
# ONO
###############################################################################
#: Monitoring related services for ONO's CronSync service.
sub run {
my (
$self,
$CRON_DIR,
$job,
$data_ref,
$TYPE_ref,
$vars_ref,
$vpath,
$sec,$min,$hour,
$mday,$mon,$year,
$wday,$yday,
$timestamp,
$switches,
) = @_;
#: Execution logic.
my %data = %$data_ref;
my %TYPE = %$TYPE_ref;
my %vars = %$vars_ref;
my $db = ONO::Cron::CronSync::ToolBox->connect();
my ($DATA,$MAIL);
if ($data{'type'} eq "monitor") {
# create the monitoring dir
if (!ONO::IO->exists("var/log/monitoring/machines")) {
ONO::IO->mkpath("var/log/monitoring/machines");
}
if ($db) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database connected [OK]",$switches);
foreach my $line (ONO::DB->select($db,"ono_network","relation = 'ono' AND status > 0 AND flags LIKE '%m%'")) {
my @col = ONO::DB->readcols($line);
if ($col[26] || $col[27]) {
if (!$col[26]) {
$col[26] = $col[27];
}
my $HTML = ONO::IO->curl($col[26]);
$HTML =~ s~(\n|\r|\t)~~g;
my $LOG = "OFFLINE";
if ($HTML =~ /\<body/i && $HTML =~ /\<\/body\>/i && $HTML =~ /ono\.css/i && $HTML !~ /document has moved/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': $col[26] is ONLINE [OK]",$switches);
$LOG = "ONLINE";
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': $col[26] is OFFLINE [ERROR]",$switches);
}
ONO::ToolBox::Logfile->log("var/log/monitoring/$year$mon$mday.log","cronsync","$col[1]: $LOG [$col[26]]");
}
}
foreach my $line (ONO::DB->select($db,"ono_network","relation = 'ono' AND status > 0 AND flags LIKE '%M%'")) {
my @col = ONO::DB->readcols($line);
my $FILE = ONO::IO->load("var/community/ono/network/devices/$col[1]/syslog_exportfile.txt");
$FILE =~ s~(\n|\r|\t)~~g;
if ($FILE) {
my $HTML = ONO::IO->curl($FILE);
my $STAT = "FAILED";
if ($HTML && $HTML !~ /document has moved/ && $HTML !~ /\<html\>/ && length $HTML < 1024) {
$STAT = "SUCCESS";
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ID '$col[1]' Syslog ExportFile: $HTML",$switches);
if (!ONO::IO->exists("var/log/monitoring/syslog/$col[1]")) {
ONO::IO->mkpath("var/log/monitoring/syslog/$col[1]");
}
ONO::IO->store("var/log/monitoring/syslog/$col[1].txt",$HTML);
$HTML =~ s~;~,~g;
ONO::ToolBox::Logfile->log("var/log/monitoring/syslog/$col[1]/$year$mon$mday.log","cronsync","$col[1]: $HTML");
}
ONO::IO->store("var/log/monitoring/syslog/$col[1].status","CURL $FILE : $STAT");
}
}
my (%units,%units_current,%units_counter,%units_online,%units_offline,%units_5m,%units_1h,%sql);
foreach my $line (reverse ONO::IO->list("var/log/monitoring/$year$mon$mday.log")) {
my @lp = split(/;/,$line);
$lp[5] =~ m~^(.*?): (.*?) ~;
my $unit = $1;
my $status = $2;
$units{$unit} .= "$lp[2]:$status\n";
$units_counter{$unit}++;
if ($status eq "ONLINE") {
$units_online{$unit}++;
}
if ($status eq "OFFLINE") {
$units_offline{$unit}++;
}
if ($lp[2] =~ /^$hour$min/) {
$units_current{$unit} = $status;
}
if ($units_counter{$unit} < 6 && $status eq "ONLINE") {
$units_5m{$unit}++;
}
if ($units_counter{$unit} < 61 && $status eq "ONLINE") {
$units_1h{$unit}++;
}
}
foreach my $unit (keys %units) {
if (!$units_current{$unit}) {
$units_current{$unit} = "OFFLINE";
}
if (!$units_online{$unit}) {
$units_online{$unit} = 0;
}
if (!$units_offline{$unit}) {
$units_offline{$unit} = 0;
}
my ($per_on,$per_off) = (0,0);
my ($per_5m,$per_1h) = (0,0);
if ($units_5m{$unit}) {
$per_5m = $units_5m{$unit};
}
if ($units_1h{$unit}) {
$per_1h = $units_1h{$unit};
}
if ($units_counter{$unit}) {
$per_on = (int(1000/$units_counter{$unit}*$units_online{$unit}))/10;
$per_off = (int(1000/$units_counter{$unit}*$units_offline{$unit}))/10;
}
my $sql_status = "ERROR";
my $DATA = "CURRENT:$units_current{$unit}\nCOUNTER:$units_counter{$unit}\nONLINE:$units_online{$unit}\nOFFLINE:$units_offline{$unit}\n";
$DATA .= "ONLINE_PERCENT:$per_on\nOFFLINE_PERCENT:$per_off\nONLINE_5MIN:$per_5m\nONLINE_1HOUR:$per_1h\nSQL_STATUS:$sql_status\n";
ONO::IO->store(
"var/log/monitoring/machines/$unit.txt",
"$DATA$units{$unit}",
);
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: no database connection [ABORT]",$switches);
}
}
if ($data{'type'} eq "monitor_syslog") {
# system load and memory tests
my $cores = ONO::Core::HostOS->cpu_cores;
my ($load1,$load2,$load3) = ONO::Core::HostOS->cpu_load;
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': CPU: $cores,$load1,$load2,$load3",$switches);
my ($mem,$used) = ONO::Core::HostOS->mem_load;
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': MEM: $mem,$used",$switches);
# test if the database is really running
my $SQL_TEST = "ERROR";
my $sql = 0;
my $ID = ONO::Lib::Code::RandomID->make;
if ($db) {
ONO::DB->command($db,"INSERT INTO ono_test (id_10) VALUES ('$ID');");
foreach my $line (ONO::DB->select($db,"ono_test","id_10 = '$ID'")) {
my @col = ONO::DB->readcols($line);
$SQL_TEST = "OK";
$sql = 1;
}
ONO::DB->command($db,"DELETE FROM ono_test;");
}
# create the syslog
unless (ONO::IO->exists("var/log/syslog")) {
ONO::IO->mkdir("var/log",777);
ONO::IO->mkdir("var/log/syslog",777);
}
ONO::ToolBox::Logfile->log("var/log/syslog/$year$mon$mday.log","syslog","CPU/$cores: $load1,$load2,$load3; MEM: $mem,$used; SQL: $SQL_TEST");
if (ONO::IO->exists("etc/syslog.conf")) {
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': found: /etc/syslog.conf",$switches);
foreach my $line (ONO::IO->list("etc/syslog.conf")) {
if ($line =~ m~^ExportFile (.*)$~) {
my $FILE = $1;
$FILE =~ s~[^A-Za-z0-9\.\-\_\/]~~g;
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': export to file: $FILE",$switches);
ONO::IO->store($FILE,"$timestamp;$year$mon$mday;$hour$min$sec;;syslog;$cores,$load1,$load2,$load3,$mem,$used,$sql;;");
}
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print ("Job '$data{'title'}': missing: /etc/syslog.conf",$switches);
}
}
if ($data{'type'} eq "monitor_export") {
my $db = ONO::Cron::CronSync::ToolBox->connect();
if ($db) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database connected [OK]",$switches);
if (!ONO::IO->exists("monitoring/advanced")) {
ONO::IO->mkdir("monitoring");
ONO::IO->mkdir("monitoring/advanced");
}
my ($BLK_ref,$UCBLK_ref) = ONO::Lib::Lang::BLK->get("en",$vars_ref);
ONO::IO->mkpath("monitoring/advanced");
ONO::IO->store("monitoring/index.shtml","<html><head></head><body></body></html>");
ONO::IO->store("monitoring/advanced/index.shtml","<html><head></head><body></body></html>");
my $CSSJS = ONO::Render->cssjs();
my %setup;
$setup{'sql_table'} = "ono_network";
$setup{'sql_col_prefix'} = "net";
$setup{'sql_relfield'} = "relation";
$setup{'sql_relation'} = "ono";
my $HTML = ONO::FW::IBS::Network->monitor(
$db,
"ono",
"/ono/admin/",
"en",
\%setup,
$vars_ref,
$vars_ref,
$BLK_ref,
);
ONO::IO->store(
"monitoring/export.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
$setup{'mini_monitoring'}++;
my $HTML = ONO::FW::IBS::Network->monitor(
$db,
"ono",
"/ono/admin/",
"en",
\%setup,
$vars_ref,
$vars_ref,
$BLK_ref,
);
ONO::IO->store(
"monitoring/export-mini.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
$setup{'mini_monitoring'} = 0;
$setup{'advanced_monitoring'}++;
my $HTML = ONO::FW::IBS::Network->monitor(
$db,
"ono",
"/ono/admin/",
"en",
\%setup,
$vars_ref,
$vars_ref,
$BLK_ref,
);
my $KEY;
if ($data{'export_advanced'}) {
$KEY = "-$data{'export_advanced'}";
}
ONO::IO->store(
"monitoring/advanced/export$KEY.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
ONO::IO->store(
"monitoring/advanced/export$KEY-$year$mon$mday.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
$HTML
</body>
</html>
~,
);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: no database connection [ABORT]",$switches);
}
}
if ($data{'type'} eq "monitor_backups") {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': $TYPE{$data{'type'}} ($data{'type'})",$switches);
my $daysecs = 24*60*60;
my $daysecs_plus = 26*60*60; # allow 2 hours to complete a backup
my ($REPORT,$STATUS);
my $STATUS_LED = "red";
foreach my $line (split(/\<br\>/,$data{'mon_config'})) {
if ($line !~ /^\#/) {
$REPORT .= qq~$line<br>~;
}
if ($line =~ s/\#\!//) {
my @lp = split(/:/,$line);
if ($lp[0] eq "backup") {
my $STAT = qq~<span class="lightred bold">not found</span>~;
if (ONO::Core::HostIO->exists("$lp[1]/BACKUP_TIMESTAMP.txt") || ONO::Core::HostIO->exists("$lp[1]/var/BACKUP_TIMESTAMP.txt")) {
my $time = ONO::Core::HostIO->load("$lp[1]/BACKUP_TIMESTAMP.txt");
if (!$time) {
$time = ONO::Core::HostIO->load("$lp[1]/var/BACKUP_TIMESTAMP.txt");
}
$STAT = qq~<span class="lightgreen bold">found</span>, ~;
if (length $time == 10) {
$STAT .= ONO::Lib::DateTime::ToolBox->get($time,0,"iz","en");
}
if ($lp[2] =~ /(d|w|m|y)/) {
my $STATUS_TIME = qq~<span class="lightgreen bold">OK</span>~;
if (
($lp[2] =~ /d/ && $time < ($timestamp - $daysecs_plus)) ||
($lp[2] =~ /D/ && $time < ($timestamp - $daysecs_plus*3)) ||
($lp[2] =~ /w/ && $time < ($timestamp - $daysecs*7)) ||
($lp[2] =~ /m/ && $time < ($timestamp - $daysecs*31)) ||
($lp[2] =~ /y/ && $time < ($timestamp - $daysecs*366))
) {
$STATUS_TIME = qq~<span class="lightred bold">OUTDATED</span>~;
$STATUS = "ERROR";
}
$STAT .= " [$STATUS_TIME]";
}
} else {
$STATUS = "ERROR";
}
$REPORT .= qq~<span class="bold">$lp[3]</span>: $STAT<br><span class="col9">\+ $lp[1]</span><br>~;
if ($lp[2] !~ /Q/) {
my $SQL_FILE = "$lp[1]/var/BACKUP_MYSQLDUMP.sql";
if (!ONO::Core::HostIO->exists($SQL_FILE)) {
foreach my $test (ONO::Core::HostIO->ls($lp[1])) {
if ($test !~ /^\./ && $test =~ /[A-Za-z0-9]/ && ONO::Core::HostIO->exists("$lp[1]/$test/latest/sql")) {
foreach my $sql (ONO::Core::HostIO->ls("$lp[1]/$test/latest/sql")) {
if ($sql !~ /^\./ && $sql =~ /[A-Za-z0-9]/ && $sql =~ /\.sql$/) {
$SQL_FILE = "$lp[1]/$test/latest/sql/$sql";
}
}
}
}
}
if (ONO::Core::HostIO->exists($SQL_FILE)) {
my $size = ONO::Core::HostIO->size($SQL_FILE);
my $time = ONO::Core::HostIO->timestamp($SQL_FILE);
my $unit = "b";
if ($size > 8192) {
$size = 1+int($size/1024);
$unit = "kb";
if ($size > 8192) {
$size = 1+int($size/1024);
$unit = "MB";
}
}
my $STATUS_TIME = qq~<span class="lightred">UNKNOWN</span>~;
if (length $time == 10) {
if (
($lp[2] =~ /d/ && $time < ($timestamp - $daysecs)) ||
($lp[2] =~ /w/ && $time < ($timestamp - $daysecs*7)) ||
($lp[2] =~ /m/ && $time < ($timestamp - $daysecs*31)) ||
($lp[2] =~ /y/ && $time < ($timestamp - $daysecs*366))
) {
$STATUS_TIME = qq~<span class="lightred bold">OUTDATED</span>~;
} else {
$STATUS_TIME = qq~<span class="lightgreen bold">OK</span>~;
}
$time = ONO::Lib::DateTime::ToolBox->get($time,0,"iz","en");
}
$REPORT .= qq~<span class="col9">\+ SQL backup <span class="lightgreen bold">found</span>, <span class="bold">$size</span>$unit, $time [$STATUS_TIME]</span><br>~;
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">no SQL backup found</span></span><br>~;
}
if ($lp[2] =~ /q/) {
if (ONO::Core::HostIO->exists("$lp[1]/etc/sql/ono.conf")) {
my %conf;
foreach my $line (ONO::Core::HostIO->list("$lp[1]/etc/sql/ono.conf")) {
$line =~ s~(\n|\r|\t)~~g;
$line =~ m~^(.*?) (.*?)$~;
$conf{$1} = $2;
}
my $db = ONO::DB->connect($conf{'database'},"d",$conf{'username'},$conf{'password'});
if ($db) {
my $time = ONO::DB->get($db,"value","ono_status","name = 'ono_db_timestamp'");
if (length $time == 10) {
my $STATUS_TIME;
if (
($lp[2] =~ /d/ && $time < ($timestamp - $daysecs)) ||
($lp[2] =~ /w/ && $time < ($timestamp - $daysecs*7)) ||
($lp[2] =~ /m/ && $time < ($timestamp - $daysecs*31)) ||
($lp[2] =~ /y/ && $time < ($timestamp - $daysecs*366))
) {
$STATUS_TIME = qq~<span class="lightred bold">OUTDATED</span>~;
} else {
$STATUS_TIME = qq~<span class="lightgreen bold">OK</span>~;
}
$time = ONO::Lib::DateTime::ToolBox->get($time,0,"iz","en");
$REPORT .= qq~<span class="col9">\+ SQL live backup is <span class="lightgreen bold">ONLINE</span>, updated $time [$STATUS_TIME]</span><br>~;
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">No ono_db_timestamp entry found in $conf{'database'} / ono_status</span></span><br>~;
}
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">Cannot connect to the database ($conf{'database'},$conf{'username'},PWD:XXXXXXXX)</span></span><br>~;
}
} else {
$REPORT .= qq~<span class="col9">\+ <span class="lightred bold">Missing /etc/sql/ono.conf ($lp[1]/etc/sql/ono.conf)</span></span><br>~;
}
}
}
}
if ($lp[0] eq "disk") {
foreach my $disk (ONO::IO->disks) {
my @dp = split(/\^/,$disk);
if ($lp[1] eq $dp[0] || $lp[1] eq "$dp[0]/") {
for (my $i = 1; $i < 4; $i++) {
$dp[$i] = 1+int($dp[$i]/1024/1024);
}
$dp[4] =~ s~[^0-9]~~g;
my $color = "lightgreen";
if ($dp[4] > 80 && $dp[3] < 250) {
$color = "orange";
}
if ($dp[4] > 90 && $dp[3] < 100) {
$color = "lightred";
}
if ($dp[3] > 49) {
$color = "orange";
}
if ($dp[3] > 99) {
$color = "lightgreen";
}
$REPORT .= qq~$lp[3]: <span class="$color bold">$dp[4]%</span>
(<span class="$color bold">$dp[2]</span>GB of <span class="$color bold">$dp[1]</span>GB used, <span class="$color bold">$dp[3]</span>GB free)<br>
~;
}
}
}
}
}
if (!$STATUS) {
$STATUS = "OK";
$STATUS_LED = "green";
}
$REPORT .= qq~<br><br>Report has been generated on $year/$mon/$mday \@ $hour:$min:$sec~;
my $CSSJS = ONO::Render->cssjs();
my $REPORT_MAIL = $REPORT;
my $REPORT_HTML = $REPORT;
$REPORT_HTML =~ s~\+ ~ \+ ~g;
ONO::IO->mkpath("monitoring/backups");
ONO::IO->store("monitoring/index.shtml","<html><head></head><body></body></html>");
ONO::IO->store("monitoring/backups/index.shtml","<html><head></head><body></body></html>");
my $KEY;
if ($data{'mon_file'} =~ /[A-Za-z0-9]/) {
$KEY = "-$data{'mon_file'}";
}
ONO::IO->store(
"monitoring/backups/export$KEY.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
<div class="p20">
<h1>$data{'mon_title'} \[$STATUS\]</h1>
<div class="mb10">$year/$mon/$mday \@ $hour:$min:$sec</div>
$REPORT
</div>
</body>
</html>
~,
);
ONO::IO->store(
"monitoring/backups/export$KEY-mini.html",
qq~<html>
<head>
<meta charset="utf-8">
<meta name="robots" content="noindex, nofollow">
<meta http-equiv="refresh" content="20">
$CSSJS
</head>
<body>
<table class="default_table auto">
<tr>
<td><img class="block12 auto" src="/ono/osr/images/leds/16x16/$STATUS_LED.png" alt=""></td>
<td class="small">$data{'mon_title'} \[$STATUS\]</td>
</tr>
</table>
</body>
</html>
~,
);
if ($data{'mon_email_hh'} && $data{'mon_email_hh'} == $hour && $data{'mon_email_mm'} && $data{'mon_email_mm'} == $min) {
$REPORT_MAIL =~ s~\ \;~ ~g;
$REPORT_MAIL =~ s~\<br\>~\n~g;
ONO::ToolBox::SendMail->sendmail(
"mail:noreply",
"$data{'mon_email'}",
"[ONO] $data{'mon_title'} \[$STATUS\]",
"$REPORT_MAIL\n\n",
);
}
}
return ($DATA,$MAIL);
}
###############################################################################
# end of script
###############################################################################
1;
__END__