ONO::Cron::CronSync::OS
package ONO::Cron::CronSync::OS;
################################################################################
# COPYRIGHT / LICENSE #
################################################################################
#
# This file is part of the ONO Software Project.
#
# Copyright (C) 2000-2025 Jos KIRPS [ www.kirps.com | jos_AT_kirps_DOT_com ]
# and The Joopita Project [ www.joopita.org | contact_AT_joopita_DOT_com ]
#
# This file, as well as other parts of the ONO Software Project or related
# elements, are FREE SOFTWARE available under the ARTISTIC LICENSE 2.0.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# For the full license, see /ono/osr/license/LICENSE.txt, or write to
# jos_AT_kirps_DOT_com or contact_AT_joopita_DOT_com.
#
################################################################################
# END OF COPYRIGHT / LICENSE, HERE COMES THE CODE ... #
################################################################################
use strict;
use ONO::IO;
use ONO::Lib::Basic;
use ONO::ToolBox::System;
use ONO::Cron::CronSync::ToolBox;
use ONO::Core::HostIO;
use ONO::Core::HostOS;
use ONO::DB;
use ONO::ToolBox::SendMail;
use ONO::ToolBox::Test;
use ONO::ToolBox::ONO;
use ONO::Lib::DateTime::ToolBox;
use ONO::ToolBox::Analytics;
use ONO::ToolBox::Security;
use ONO::Lib::Web::BotDetect;
###############################################################################
# ONO
###############################################################################
#: Operating System related services for ONO's CronSync service.
sub run {
my (
$self,
$CRON_DIR,
$job,
$data_ref,
$TYPE_ref,
$vars_ref,
$vpath,
$sec,$min,$hour,
$mday,$mon,$year,
$wday,$yday,
$timestamp,
$switches,
) = @_;
#: Execution logic.
my %data = %$data_ref;
my %TYPE = %$TYPE_ref;
my %vars = %$vars_ref;
my ($DATA,$MAIL);
if ($data{'type'} eq "os_security") {
if ($data{'os_security_system_users'}) {
if ($min%$data{'os_security_system_users'} == 0) {
my ($SITE,$DOMAIN,$FULLDOM) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN) {
my ($users,$USERS);
foreach my $who (ONO::Core::HostOS->who) {
if ($who) {
$users++;
$USERS .= "$who\n";
}
}
if ($users) {
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] LOGIN DETECTED ($users users, $hour:$min:$sec)",
qq~The following users are logged in:\n\n$USERS~,
);
}
}
}
}
if ($data{'os_security_apache_ip'}) {
if ($min%$data{'os_security_apache_ip'} == 0) {
my ($SITE,$DOMAIN,$FULLDOM) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN) {
$vars{'filter'} = "undef";
$vars{'ip_analytics_min_hits'} = $data{'os_security_apache_ip_hits'};
$vars{'ip_analytics_mode'} = "mail";
my %RULES;
foreach my $line (ONO::IO->list("etc/security/iptables_rules.txt")) {
if ($line =~ m~^(.*?):(.*?):~) {
$RULES{$1} = $2;
}
}
my ($LIST_ref) = ONO::ToolBox::Security->apache_log_ip_analytics(\%RULES,\%vars);
my @LIST = @$LIST_ref;
my $REP = $LIST[32];
if ($REP && $REP ne ONO::IO->load("var/tmp/security/os_security_apache_ip_mail.txt")) {
ONO::IO->mkpath("var/tmp/security");
ONO::IO->store("var/tmp/security/os_security_apache_ip_mail.txt",$REP);
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] SUSPICIOUS APACHE IP DETECTED",
qq~The following IPs have been detected:\n\nhttps://$FULLDOM/ono/admin/\?/info_security/\&filter=undef\n\n$REP~,
);
}
}
}
}
}
if ($data{'type'} eq "os_monitor") {
my ($errors,$REPORT) = ONO::ToolBox::Test->media_processors("H");
if ($errors) {
my ($SITE,$DOMAIN) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN) {
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] OS SERVICE MONITOR ERRORS ($errors)",
qq~The following errors have been reported:\n\n$REPORT~,
);
}
}
}
if ($data{'type'} eq "os_control") {
}
if ($data{'type'} eq "os_diskfree") {
my ($DISK,@disks);
foreach my $disk (ONO::IO->disks) {
$DISK .= $disk;
@disks = (@disks,$disk);
}
my ($REP,$MAX) = ONO::ToolBox::System->diskfree_display(\@disks,"Am");
$MAIL .= $REP;
if ($data{'os_diskfree_mail'} =~ /^(.*?)\@(.*?)\.(.*?)$/) {
my ($SITE,$DOMAIN) = ONO::Cron::CronSync::ToolBox->get_site();
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] Disk Usage Report [Max:$MAX\%]",
$MAIL,
);
}
ONO::IO->store("$CRON_DIR/diskfree.txt",$DISK);
}
if ($data{'type'} eq "os_chmod") {
if ($data{'os_chmod_logs'}) {
foreach my $file ('/var/log/apache2/access_log','/var/log/apache2/error_log','/var/log/apache2/access.log','/var/log/apache2/error.log') {
if (-e $file) {
$DATA .= ONO::IO->exec("chmod 644 $file");
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': chmod 644 $file",$switches);
}
}
}
if ($data{'os_chmod_www'} || $data{'os_chown_www'}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting chmod or chown process",$switches);
foreach my $dir (ONO::Core::HostIO->ls("/var/www")) {
if ($dir !~ /^\./ && $dir =~ /[A-Za-z0-9]/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': - Processing web '$dir' ...",$switches);
foreach my $subdir (ONO::Core::HostIO->ls("/var/www/$dir/html")) {
if ($subdir !~ /^\./ && $subdir =~ /[A-Za-z0-9]/ && $subdir !~ /^(etc)$/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': -- /var/www/$dir/html/$subdir ...",$switches);
if ($data{'os_chmod_www'}) {
ONO::Core::HostIO->chmodr("/var/www/$dir/",777);
}
if ($data{'os_chown_www'}) {
ONO::Core::HostIO->chowngrpr("/var/www/$dir/",$dir,$dir);
}
}
}
}
}
}
}
if ($data{'type'} eq "os_readconf") {
ONO::IO->exec("mkdir /etc/ono");
ONO::IO->exec("mkdir /etc/ono/readconf");
ONO::IO->exec("mkdir /etc/ono/readconf/etc");
unless (-e "/etc/ono/readconf/etc") {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR - /etc/ono/readconf/etc could not be created",$switches);
}
my %used;
foreach my $opt ('system','webserver','database','fileserver','mailserver') {
if ($data{"os_readconf_$opt"}) {
my @objs;
if ($opt eq "system") {
@objs = ('crontab','hostname','hosts','passwd');
}
if ($opt eq "database") {
@objs = ('mysql');
}
if ($opt eq "webserver") {
@objs = ('apache2','httpd');
}
if ($opt eq "fileserver") {
@objs = ('samba');
}
if ($opt eq "mailserver") {
@objs = ('postfix','dovecot','courier','mail','passwd','spamassassin','default/saslauthd','default/spamassassin');
}
foreach my $obj (@objs) {
if (-e "/etc/$obj" && !$used{$obj}) {
$used{$obj}++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Copying /etc/$obj to /etc/ono/readconf/etc/$obj on core system",$switches);
ONO::IO->exec("cp -r /etc/$obj /etc/ono/readconf/etc/");
ONO::IO->exec("chmod -R 755 /etc/ono/readconf/etc/$obj");
ONO::Core::HostIO->store("/etc/ono/readconf/etc/$obj.timestamp",$timestamp);
}
}
}
}
}
if ($data{'type'} eq "os_checkconf") {
my $COM = "cp -r /etc/apache2 $vpath/var/hostinfo/etc/";
ONO::IO->rmdir("var/hostinfo/etc");
ONO::IO->mkpath("var/hostinfo/etc");
ONO::IO->store("var/hostinfo/checkconf_report.txt","$COM\n");
ONO::IO->exec($COM);
ONO::IO->chmod("var/hostinfo/etc","777");
}
if ($data{'type'} eq "os_log_analyzer" && ONO::IO->sysload(50)) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting OS Log Analyzer",$switches);
ONO::Core::HostIO->cp("/var/log/apache2/access.log","$vpath/var/log/security/apache_access_log.txt");
ONO::Core::HostIO->cp("/var/log/apache2/access.log.1","$vpath/var/log/security/apache_access_log_previous.txt");
my (%STAT,%REFERERS,%AGENTS,%AUTH);
foreach my $opt ("/var/log/apache2/access.log","/var/log/apache2/access.log.1") {
if (ONO::Core::HostIO->exists($opt)) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Analyzing $opt",$switches);
my $WARNINGS;
foreach my $line (ONO::Core::HostIO->list($opt)) {
if ($line =~ m~(.*?) (.*?) \[(.*?)\] (.*?) "(.*?) (.*?) HTTP/(.*?)" (.*?) (.*?) "(.*?)" "(.*?)"~) {
my $domain = $1;
my $IP_32 = $2;
my $time = $3;
my $timestamp = $4;
my $method = $5;
my $url = $6;
my $referer = $10;
my $agent = $11;
$IP_32 =~ m~^(.*?)\.(.*?)\.(.*?)\.(.*?)$~;
my $IP_08 = $1;
my $IP_16 = "$1.$2";
my $IP_24 = "$1.$2.$3";
$STAT{"apache_ip_08_$IP_08"}++;
$STAT{"apache_ip_16_$IP_16"}++;
$STAT{"apache_ip_24_$IP_24"}++;
$STAT{"apache_ip_32_$IP_32"}++;
$AGENTS{"$IP_32=$agent"}++;
# $REFERERS{"$IP_32=$referer"}++;
}
# warn if evil bots, trackers or apps have been detected
if ($line =~ m~(copytrack|plaghunter|photoclaim|pixsy)~gi) {
ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Potentially evil access detected: $1",$switches);
$WARNINGS .= qq~$1 - $line~;
}
}
if ($WARNINGS) {
ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Warnings have been detected !",$switches);
my ($SITE,$DOMAIN,$FULLDOM) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN && $min%30 == 0) {
ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Trying to send warnings to the tracker mail",$switches);
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] WARNING - EVIL WEB ACCESS DETECTED",
qq~The following access.log entries look suspicious:\n\n$WARNINGS~,
);
}
}
}
}
foreach my $opt ("/var/log/auth.log","/var/log/auth.log.1") {
if (ONO::Core::HostIO->exists($opt)) {
foreach my $line (ONO::Core::HostIO->list($opt)) {
$line =~ s~(\n|\r|\t)~~g;
if ($line =~ / fail/gi) {
if ($line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+) ~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
if ($line =~ /root/ && $line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+) ~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
if ($line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+)$~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
if ($line =~ /root/ && $line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+)$~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
}
}
}
}
ONO::IO->mkpath("var/log/security");
my ($REPORT,$REPORT_AUTH,$REFS,$AGTS);
foreach my $key (reverse sort {$STAT{$a} <=> $STAT{$b}} keys %STAT) {
$REPORT .= qq~$key=$STAT{$key}\n~;
}
ONO::IO->store("var/log/security/report_ips.txt",$REPORT);
ONO::IO->store("var/log/security/report_ips_$year$mon$mday.txt",$REPORT);
foreach my $key (reverse sort {$REFERERS{$a} <=> $REFERERS{$b}} keys %REFERERS) {
$REFS .= qq~$key=$REFERERS{$key}\n~;
}
ONO::IO->store("var/log/security/report_referers.txt",$REFS);
foreach my $key (reverse sort {$AGENTS{$a} <=> $AGENTS{$b}} keys %AGENTS) {
$AGTS .= qq~$key=$AGENTS{$key}\n~;
}
ONO::IO->store("var/log/security/report_agents.txt",$AGTS);
foreach my $key (reverse sort {$AUTH{$a} <=> $AUTH{$b}} keys %AUTH) {
$REPORT_AUTH .= qq~$key=$AUTH{$key}\n~;
}
ONO::IO->store("var/log/security/report_ips_auth.txt",$REPORT_AUTH);
ONO::IO->store("var/log/security/report_ips_auth_$year$mon$mday.txt",$REPORT_AUTH);
}
if ($data{'type'} eq "os_firewall" && ONO::IO->sysload(50)) {
my (%FRIENDLY,$force_refresh);
foreach my $line (ONO::IO->list("etc/security/friendly_ips.txt")) {
$line =~ s~[^0-9\.]~~g;
$FRIENDLY{$line}++;
}
if ($data{'os_firewall_refresh'} > 0 && $data{'os_firewall_refresh'} eq $hour) {
if (!ONO::IO->exists("var/log/security/firewall_refresh_$year$mon$mday.txt")) {
ONO::IO->store("var/log/security/firewall_refresh_$year$mon$mday.txt","$hour:$min:$sec");
$force_refresh++;
}
}
if ($data{'os_firewall_auth_threshold'} > 0 || $data{'os_firewall_bad_script_threshold'} > 0 || $data{'os_firewall_bad_bot_threshold'} > 0) {
if (!ONO::IO->exists("var/log/security/firewall_tmp_flush_$year$mon$mday.txt")) {
ONO::IO->rm("etc/security/iptables_rules_tmp.txt");
ONO::IO->store("var/log/security/firewall_tmp_flush_$year$mon$mday.txt","$hour:$min:$sec");
}
my (%block,%used,%script,%bot);
if ($data{'os_firewall_auth_threshold'} > 0) {
foreach my $line (ONO::IO->list("var/log/security/report_ips_auth.txt")) {
$line =~ s~(\n|\r|\t)~~g;
if ($line =~ /^auth_ip_(.*?)=(.*?)$/) {
if (1+$2 > $data{'os_firewall_auth_threshold'} && !ONO::IO->is_local_ip($1)) {
$block{$1}++;
}
}
}
}
if ($min%5 == 0 && ($data{'os_firewall_bad_script_threshold'} > 0 || $data{'os_firewall_bad_bot_threshold'} > 0)) {
foreach my $line (ONO::IO->list("var/log/security/apache_access_log.txt")) {
if ($data{'os_firewall_bad_script_threshold'} > 0) {
if ($line =~ /\.php /i || $line =~ /wp-config\.php/i) {
if ($line =~ /^(.*?) (.*?)\.(.*?)\.(.*?)\.(.*?) \[/) {
my $ip = "$2.$3.$4.$5";
$script{$ip}++;
if ($script{$ip}+1 > $data{'os_firewall_bad_script_threshold'}) {
$block{$ip}++;
}
}
}
}
if ($data{'os_firewall_bad_bot_threshold'} > 0) {
# ONO::Lib::Web::BotDetect - first check if line even applies (faster...)
if (ONO::Lib::Web::BotDetect->evil_bot($line)) {
if ($line =~ /^(.*?) (.*?)\.(.*?)\.(.*?)\.(.*?) \[(.*?)\] (.*?) "(.*?)" (.*?) (.*?) "(.*?)" "(.*?)"/) {
my $ip = "$2.$3.$4.$5";
my $agent = $12;
# ONO::Lib::Web::BotDetect - then make sure it's really the user agent that's bad
if (ONO::Lib::Web::BotDetect->evil_bot($agent)) {
$bot{$ip}++;
if ($bot{$ip}+1 > $data{'os_firewall_bad_bot_threshold'}) {
$block{$ip}++;
}
}
}
}
}
}
}
foreach my $line (ONO::IO->list("etc/security/iptables_rules_tmp.txt")) {
my @lp = split(/\:/,$line);
if ($lp[1] eq "block") {
$used{$lp[0]}++;
}
}
foreach my $key (keys %block) {
if ($key =~ /^(.*?)\.(.*?)\.(.*?)\.(.*?)$/ && !$used{$key} && !$FRIENDLY{$key} && !ONO::IO->is_local_ip($key)) {
ONO::IO->append("etc/security/iptables_rules_tmp.txt","$key:block:\n");
`/sbin/iptables -A INPUT -s $key -j DROP`;
}
}
ONO::IO->chmod("etc/security/iptables_rules_tmp.txt","777");
}
if ($data{'os_firewall_auto_release_percent'} > 0) {
if (!ONO::IO->exists("var/log/security/firewall_release_$year$mon$mday.txt")) {
my ($DATA,$REL);
foreach my $line (ONO::IO->list("etc/security/iptables_rules.txt")) {
if ($line =~ m~(.*?)\.(.*?)\.(.*?)\.(.*?):block:~) {
if (ONO::Lib::Basic->random_percent($data{'os_firewall_auto_release_percent'})) {
$REL .= $line;
$force_refresh++;
} else {
$DATA .= $line;
}
} else {
$DATA .= $line;
}
}
ONO::IO->mv("etc/security/iptables_rules.txt","etc/security/iptables_rules_backup/$year$mon$mday$hour.txt");
ONO::IO->store("etc/security/iptables_rules.txt",$DATA);
ONO::IO->chmod("etc/security/iptables_rules.txt","777");
ONO::IO->store("var/log/security/firewall_release_$year$mon$mday.txt",$REL);
ONO::IO->chmod("var/log/security/firewall_release_$year$mon$mday.txt","777");
}
}
if (($min%5 == 0 && ONO::IO->exists("etc/security/iptables_rules_refresh.txt")) || $force_refresh) {
my ($DATA,$EXEC,%used,@BLOCK);
my $TRAP1 = substr(time(),0,4);
my $TRAP2 = $TRAP1 - 1;
`/sbin/iptables -F`;
`/sbin/iptables -N LOG_DROP`;
`/sbin/iptables -A LOG_DROP -j LOG --log-level warning --log-prefix "INPUT-DROP:"`;
`/sbin/iptables -A LOG_DROP -j DROP`;
foreach my $line (
ONO::IO->list("etc/security/iptables_rules_custom.txt"),
ONO::IO->list("etc/security/iptables_rules_tmp.txt"),
ONO::IO->list("etc/security/iptables_rules.txt"),
ONO::IO->list("iptables_rules_trap.$TRAP1.txt"),
ONO::IO->list("iptables_rules_trap.$TRAP2.txt"),
) {
my @lp = split(/\:/,$line);
if ($lp[1] eq "block" && !$FRIENDLY{$lp[0]} && !ONO::IO->is_local_ip($lp[0])) {
my @ips = split(/\./,$lp[0]);
my $num = @ips;
my ($ip,$range) = ("$lp[0].0.0.0",8);
if ($num == 2) {
($ip,$range) = ("$lp[0].0.0",16);
}
if ($num == 3) {
($ip,$range) = ("$lp[0].0",24);
}
if ($num == 4) {
($ip,$range) = ($lp[0],32);
}
if (!$used{"$ip/$range"} && !ONO::IO->is_local_ip($ip)) {
$used{"$ip/$range"}++;
@BLOCK = (@BLOCK,"$ip/$range");
}
}
}
my ($ipcount,$IPBLOCK);
foreach my $IP (@BLOCK) {
$ipcount++;
$IPBLOCK .= "$IP,";
if ($ipcount > 32) {
$ipcount = 0;
$IPBLOCK =~ s~\,$~~;
`/sbin/iptables -A INPUT -s $IPBLOCK -j LOG_DROP`;
$IPBLOCK = "";
}
}
if ($IPBLOCK) {
$IPBLOCK =~ s~\,$~~;
`/sbin/iptables -A INPUT -s $IPBLOCK -j LOG_DROP`;
}
ONO::IO->store("etc/security/iptables_rules_updated.txt","$mday/$mon/$year \@ $hour:$min:$sec");
# ONO::IO->store("etc/security/iptables_rules_blocked.txt","# Generated file, don't touch!\n\n$DATA");
# ONO::IO->store("etc/security/iptables_rules_exec.txt","# Generated file, don't touch!\n\n$EXEC");
my $IPTABLES;
foreach my $line (`/sbin/iptables -L -n -v`) {
if ($line =~ /DROP/) {
$line = ONO::Lib::Basic->remove_bad_spaces($line);
my @lp = split(/ /,$line);
$IPTABLES .= qq~$lp[7]\n~;
}
}
ONO::IO->store("var/log/security/iptables_rules_current.txt",$IPTABLES);
ONO::IO->rm("etc/security/iptables_rules_refresh.txt");
}
}
if ($data{'type'} eq "os_logfile" && ONO::IO->sysload(50)) {
my %conf = ONO::IO->confread("etc/logfileanalyzer.conf");
my $FILE = ONO::Lib::DateTime::ToolBox->yesterday($year,$mon,$mday);
my $REPORT = "Logfile dispatching on $mday/$mon/$year\n";
foreach my $opt ("/var/log/apache2/access.log","/var/log/apache2/access.log.1") {
if (ONO::Core::HostIO->exists($opt) && ($opt eq "/var/log/apache2/access.log" || (!ONO::IO->exists("var/log/apache/report_$FILE.txt") && $conf{'loglockhour'} == $hour))) {
$REPORT .= "Processing $opt...\n";
my %DATA;
foreach my $line (ONO::Core::HostIO->list($opt)) {
if ($line =~ m~(.*?) (.*?) \[(.*?)\] (.*?) "(GET|POST) (.*?) HTTP/1.1" (.*?) (.*?) "(.*?)" "(.*?)"~) {
my $domain = $1;
my $IP = $2;
my $time = $3;
my $timestamp = $4;
my $url = $6;
my $agent = $10;
$time =~ m~^(.*?)/(.*?)/(.*?):(.*?):(.*?):(.*?)~;
my $mm = ONO::Lib::DateTime::ToolBox->mmm2mm($2);
$time = "$3$mm$1";
if (!ONO::ToolBox::Analytics->exclude($url,$agent)) {
$DATA{$domain} .= qq~$domain $IP $time $timestamp "$url" "$agent"\n~;
}
$DATA{'*'} .= qq~$domain $IP $time $timestamp "$url" "$agent"\n~;
}
}
foreach my $key (keys %conf) {
if ($key =~ /^logfileexportrule/) {
$conf{$key} =~ m~^(.*?):(.*?)$~;
ONO::Core::HostIO->mkdir($2);
ONO::Core::HostIO->chmod($2,777);
if ($opt eq "/var/log/apache2/access.log") {
$REPORT .= "Store: $2/current.log\n";
ONO::Core::HostIO->store("$2/current.log",$DATA{$1});
ONO::Core::HostIO->chmod("$2/current.log",777);
}
if ($opt eq "/var/log/apache2/access.log.1") {
$REPORT .= "Store: $2/FILE.log\n";
ONO::Core::HostIO->store("$2/$FILE.log",$DATA{$1});
ONO::Core::HostIO->chmod("$2/$FILE.log",777);
ONO::IO->store("var/log/apache/report_$FILE.txt",$REPORT);
}
}
}
}
}
}
if ($data{'type'} eq "os_manage") {
my $write_report;
my $db = ONO::Cron::CronSync::ToolBox->connect();
if ($db) {
my $OS_TYPE = ONO::Core::HostOS->system();
my $IS_LINUX;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': OS is $OS_TYPE",$switches);
if ($OS_TYPE =~ /(Linux|Ubuntu|UNIX)/gi) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Does the OS allow to manage the system: YES",$switches);
$IS_LINUX++;
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Does the OS allow to manage the system: NO [ABORT]",$switches);
}
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database connected",$switches);
ONO::Core::HostIO->mkdir("/etc/ono/servermanager");
if ($data{"os_manage_system"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting system tasks...",$switches);
}
if ($data{"os_manage_webserver"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting webserver tasks...",$switches);
}
if ($data{"os_manage_database"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting database tasks...",$switches);
}
if ($data{"os_manage_fileserver"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting fileserver tasks...",$switches);
}
if ($data{"os_manage_mailserver"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting mailserver tasks...",$switches);
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/courier");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/default");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/dovecot");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/dovecot/conf.d");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/mail");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/postfix");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/spamassassin");
my ($update_mailboxes,$update_domains,$update_addresses);
# mailboxes - create
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes","status = 'create'")) {
my @row = ONO::DB->readcols($line);
$update_mailboxes++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailbox '$row[2]/$row[3]' will be created...",$switches);
my $username = "ono-mail-$row[3]";
my $password = $row[4];
if ($password && !ONO::Core::HostIO->userget($username) && $IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Creating UNIX user '$row[3]' for Mailbox '$row[1]' ...",$switches);
if (!ONO::Core::HostIO->groupget("ono-mail")) {
ONO::Core::HostIO->groupadd("ono-mail");
}
ONO::Core::HostIO->useradd($username,$password,"ono-mail","/var/mail/$username","/bin/false");
ONO::Core::HostIO->mkdir("/var/mail/$username");
ONO::Core::HostIO->chown("/var/mail/$username",$username);
ONO::Core::HostIO->chgrp("/var/mail/$username","ono-mail");
ONO::Core::HostIO->chmod("/var/mail/$username","0750");
}
ONO::DB->command($db,"UPDATE ono_servermanager_mail_mailboxes SET status = '', password = '' WHERE id_10 = '$row[1]'");
}
# mailboxes - update
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes","status = 'update'")) {
my @row = ONO::DB->readcols($line);
$update_mailboxes++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailbox '$row[2]/$row[3]' will be updated...",$switches);
my $username = "ono-mail-$row[3]";
my $password = $row[4];
# add user if missing for some obscure reason...
if (!ONO::Core::HostIO->userget($username)) {
ONO::Core::HostIO->useradd($username,$password,"ono-mail","/var/mail/$username","/bin/false");
ONO::Core::HostIO->mkdir("/var/mail/$username");
ONO::Core::HostIO->chown("/var/mail/$username",$username);
ONO::Core::HostIO->chgrp("/var/mail/$username","ono-mail");
ONO::Core::HostIO->chmod("/var/mail/$username","0750");
}
# change user
if ($password && ONO::Core::HostIO->userget($username) && $IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Updating UNIX user '$username'...",$switches);
ONO::Core::HostIO->userpwd($username,$password);
}
ONO::DB->command($db,"UPDATE ono_servermanager_mail_mailboxes SET status = '' WHERE id_10 = '$row[1]'");
}
# mailboxes - delete
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes","status = 'delete'")) {
my @row = ONO::DB->readcols($line);
$update_mailboxes++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailbox '$row[2]/$row[3]' will be deleted...",$switches);
my $username = "ono-mail-$row[3]";
if (ONO::Core::HostIO->userget($username) && $IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Deleting UNIX user '$username'...",$switches);
ONO::Core::HostIO->userdel($username);
}
ONO::DB->command($db,"DELETE FROM ono_servermanager_mail_mailboxes WHERE id_10 = '$row[1]'");
}
# domains - create
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = 'create'")) {
my @row = ONO::DB->readcols($line);
$update_domains++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domain '$row[2]' will be created...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_domains SET status = '' WHERE id_10 = '$row[1]'");
}
# domains - update
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = 'update'")) {
my @row = ONO::DB->readcols($line);
$update_domains++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domain '$row[2]' will be updated...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_domains SET status = '' WHERE id_10 = '$row[1]'");
}
# domains - delete
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = 'delete'")) {
my @row = ONO::DB->readcols($line);
$update_domains++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domain '$row[2]' will be deleted...",$switches);
ONO::DB->command($db,"DELETE FROM ono_servermanager_mail_domains WHERE id_10 = '$row[1]'");
}
# addresses - create
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = 'create'")) {
my @row = ONO::DB->readcols($line);
$update_addresses++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail '$row[2]\@$row[3]' will be created...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_addresses SET status = '' WHERE id_10 = '$row[1]'");
}
# addresses - update
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = 'update'")) {
my @row = ONO::DB->readcols($line);
$update_addresses++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail '$row[2]\@$row[3]' will be updated...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_addresses SET status = '' WHERE id_10 = '$row[1]'");
}
# addresses - delete
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = 'delete'")) {
my @row = ONO::DB->readcols($line);
$update_addresses++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail '$row[2]\@$row[3]' will be deleted...",$switches);
ONO::DB->command($db,"DELETE FROM ono_servermanager_mail_addresses WHERE id_10 = '$row[1]'");
}
# get data
my (%accounts,%domains);
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains")) {
my @row = ONO::DB->readcols($line);
$domains{$row[1]} = $row[2];
}
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes")) {
my @row = ONO::DB->readcols($line);
$accounts{$row[1]} = $row[3];
}
# updates mailboxes
if ($update_mailboxes) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailboxes have been updated!",$switches);
}
# updates domains
if ($update_domains) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domains have been updated!",$switches);
my $DATA;
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = '' OR status IS NULL")) {
my @row = ONO::DB->readcols($line);
if ($row[2] =~ /^(.*?)\.(.*?)$/) {
$DATA .= "$row[2] ";
}
}
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/domains.txt",$DATA);
}
# updates addresses
if ($update_addresses) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail addresses have been updated!",$switches);
my ($ADDR,%addresses);
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = '' OR status IS NULL")) {
my @row = ONO::DB->readcols($line);
if ($row[4] =~ /[A-Za-z0-9]/) {
if ($row[4] =~ /^(.*?)\@(.*?)\.(.*?)$/) {
$addresses{"$row[2]\@$domains{$row[3]}"} .= "$row[4] ";
} else {
$addresses{"$row[2]\@$domains{$row[3]}"} .= "ono-mail-$accounts{$row[4]} ";
}
}
}
my $count = 0;
foreach my $key (sort keys %addresses) {
$count++;
$ADDR .= "$key $addresses{$key}\n";
}
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Putting $count e-mail addresses into the Postfix virtual file",$switches);
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/addresses.txt",$ADDR);
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/postfix/virtual",$ADDR);
}
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database analysis and user updates completed...",$switches);
if ($update_mailboxes) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update mailboxes? YES",$switches);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update mailboxes? NO",$switches);
}
if ($update_domains) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update domains? YES",$switches);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update domains? NO",$switches);
}
if ($update_addresses) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update e-mail addresses? YES",$switches);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update e-mail addresses? NO",$switches);
}
# set this to 1 in order to always test updating the server config files:
my $force_test = 0;
# my $force_test = 1;
if ($force_test) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': WARNING! WE'RE RUNNING IN FORCE TEST MODE, NOT SUITED FOR PRODUCTION SYSTEMS !!!",$switches);
}
# generate server files
if ($force_test || $update_mailboxes || $update_domains || $update_addresses) {
$write_report++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Reading site setup...",$switches);
my %site = ONO::ToolBox::ONO->get();
my $HOSTNAME = $site{'domain'};
my $DOMAIN = $site{'domain'};
$DOMAIN =~ s~^(.*)\.(.*?)\.(.*?)$~$2\.$3~;
my $IP = $site{'ip'};
my $MAIL_DOMAIN_LIST = ONO::Core::HostIO->load("/etc/ono/servermanager/mail/domains.txt");
if ($HOSTNAME && $DOMAIN && $IP && $MAIL_DOMAIN_LIST) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Server Hostname is '$HOSTNAME'.",$switches);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Server Domain is '$DOMAIN'.",$switches);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Server IP is '$IP'.",$switches);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Generating server files...",$switches);
my @server_files = (
'dovecot/dovecot.conf',
'dovecot/conf.d/10-auth.conf',
'dovecot/conf.d/10-director.conf',
'dovecot/conf.d/10-logging.conf',
'dovecot/conf.d/10-mail.conf',
'dovecot/conf.d/10-master.conf',
'dovecot/conf.d/10-ssl.conf',
'dovecot/conf.d/10-tcpwrapper.conf',
'dovecot/conf.d/15-lda.conf',
'dovecot/conf.d/15-mailboxes.conf',
'dovecot/conf.d/20-imap.conf',
'dovecot/conf.d/90-acl.conf',
'dovecot/conf.d/90-plugin.conf',
'dovecot/conf.d/90-quota.conf',
'spamassassin/local.cf',
'mail/sendmail.conf',
'postfix/main.cf',
'postfix/master.cf',
'default/saslauthd',
'default/spamassassin',
);
foreach my $file (@server_files) {
my $DATA;
foreach my $line (ONO::IO->list("ono/sys/ONO/Resources/ServerConf/etc/$file")) {
if ($line !~ /^#/) {
$line =~ s~###MAIL_HOSTNAME###~$HOSTNAME~gi;
$line =~ s~###MAIL_DOMAIN###~$DOMAIN~gi;
$line =~ s~###MAIL_IP###~$IP~gi;
$line =~ s~###MAIL_SSL###~postfix~gi;
$line =~ s~###MAIL_DOMAINS###~$MAIL_DOMAIN_LIST~gi;
}
$DATA .= $line;
}
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/$file",$DATA);
}
# write server configs to /etc (do this only on Linux, not on macOS)
if ($IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': We're running on Linux [OK]",$switches);
my $valid_conf;
foreach my $line (ONO::Core::HostIO->list("/etc/ono/servermanager.conf")) {
if ($line =~ m/^writesystemfiles true/) {
$valid_conf++;
}
}
if ($valid_conf) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': /etc/ono/servermanager.conf looks good...",$switches);
# add the postfix virtual file to the list, as this file contains all e-mail addresses to be used
@server_files = (@server_files,'postfix/virtual');
# install all server configuration files (dovecot, sendmail, postfix, and saslauthd
foreach my $file (@server_files) {
if ($file !~ /^#/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Writing /etc/$file...",$switches);
ONO::Core::HostIO->cp("/etc/ono/servermanager/mail/$file","/etc/$file");
}
}
# reload / restart mail services, to make use the new settings will be used
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': reloading and restarting all mail server services...",$switches);
ONO::Core::HostIO->mailserver_reload();
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': /etc/ono/servermanager.conf not set up correctly [ABORT]",$switches);
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': This is NOT Linux [ABORT]",$switches);
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: /etc/site.conf didn't provide useful information [ABORT]",$switches);
}
}
# bounce mailbox management
#
# This requires /etc/mail/bounce.conf, including the following data:
#
# MailBoxes "/var/mail/test,/var/mail/test2,..." -> path(s) to the mailbox(es) where bounces are stored
# Sites "/var/www/html1,/var/www/html2,..." -> path(s) to the different sites on the server
#
# The target sites need to have their own /etc/mail/bouncetargets.txt, including the following data:
#
# match_string "/var/www/site/var/tmp/bouncedmails"
#
# match_string -> the sender e-mail that shall be tracked (noreply_AT_example_DOT_com, etc...)
# "/var/www/..." -> the "var/tmp/bouncedmails" directory of the target site, which MUST exist (will not be created)
my %bounce = ONO::IO->confread("etc/mail/bounce.conf");
if ($bounce{'mailboxes'} && $bounce{'sites'}) {
ONO::IO->mkdir("var/log/mail");
ONO::IO->chmod("var/log/mail",777);
ONO::IO->store("var/log/mail/bounce.txt","Mail Bounce is configured and running...\n");
my (%targets,@mails);
foreach my $mailbox (split(/,/,",$bounce{'mailboxes'},")) {
if ($mailbox =~ /[a-z]/) {
ONO::IO->append("var/log/mail/bounce.txt","Checking mailbox '$mailbox' ...\n");
foreach my $mail (ONO::Core::HostIO->ls("$mailbox")) {
if ($mail =~ /^1(.*?)\.(.*?)\.(.*)$/) {
if (!$targets{"-loaded"}) {
$targets{"-loaded"}++;
ONO::IO->append("var/log/mail/bounce.txt","Looking for target site data ...\n");
foreach my $site (split(/,/,",$bounce{'sites'},")) {
if ($site =~ /[a-z]/) {
ONO::IO->append("var/log/mail/bounce.txt","Loading target site '$site' data ...\n");
foreach my $line (ONO::Core::HostIO->list("$site/etc/mail/bouncetargets.txt")) {
$line =~ m~(.*?) \"(.*?)\"~;
$targets{$1} = $2;
@mails = (@mails,$1);
ONO::IO->append("var/log/mail/bounce.txt","Storing target: $1 leads to $2\n");
}
}
}
}
ONO::IO->append("var/log/mail/bounce.txt","Found: '$mail' ...\n");
my ($DATA,$target_found);
foreach my $line (ONO::Core::HostIO->list("$mailbox/$mail")) {
$DATA .= $line;
}
$DATA =~ s~(\n|\r|\t)~~g;
foreach my $test (@mails) {
if ($DATA =~ /$test/) {
ONO::IO->append("var/log/mail/bounce.txt","Matches: '$test', Move: '$mailbox/$mail', To: '$targets{$test}/' ...\n");
ONO::Core::HostIO->mv("$mailbox/$mail","$targets{$test}/$mail");
ONO::Core::HostIO->chmod("$targets{$test}",777);
ONO::Core::HostIO->chmod("$targets{$test}/$mail",777);
$target_found++;
}
}
if (!$target_found) {
ONO::IO->append("var/log/mail/bounce.txt","No target detected, mail '$mailbox/$mail' will be killed ...\n");
ONO::Core::HostIO->rm("$mailbox/$mail");
}
}
}
}
}
}
}
# trigger reload mailserver
if (ONO::IO->exists("$CRON_DIR/trigger/mailserver-$job.txt")) {
if ($IS_LINUX) {
ONO::Core::HostIO->mailserver_reload();
}
ONO::IO->store("$CRON_DIR/lastrun/restart-mailserver-$job.txt",$timestamp);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Removing trigger file - $CRON_DIR/trigger/mailserver-$job.txt",$switches);
ONO::IO->rm("$CRON_DIR/trigger/mailserver-$job.txt");
}
# count disk usage (run about once an hour, on an irregular basis (1 to 100) per mailbox)
# set this to 1 in order to always test counting disk usage:
my $force_test = 0;
# my $force_test = 1;
foreach my $dir (ONO::Core::HostIO->ls("/var/mail")) {
if ($dir =~ /^ono-mail-(.*?)$/ && (int(rand(100)) == 50 || $force_test)) {
my $username = $1;
my $usage = ONO::Core::HostIO->usage("/var/mail/$dir");
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Counting mailbox '$dir' size: ${usage}kb",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_mailboxes SET quotaused = '$usage' WHERE username = '$username'");
}
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: no database connection [ABORT]",$switches);
}
if ($write_report) {
ONO::Core::HostIO->store("/etc/ono/servermanager/report.txt",$DATA);
}
}
return ($DATA,$MAIL);
}
###############################################################################
# end of script
###############################################################################
1;
__END__
################################################################################
# COPYRIGHT / LICENSE #
################################################################################
#
# This file is part of the ONO Software Project.
#
# Copyright (C) 2000-2025 Jos KIRPS [ www.kirps.com | jos_AT_kirps_DOT_com ]
# and The Joopita Project [ www.joopita.org | contact_AT_joopita_DOT_com ]
#
# This file, as well as other parts of the ONO Software Project or related
# elements, are FREE SOFTWARE available under the ARTISTIC LICENSE 2.0.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# For the full license, see /ono/osr/license/LICENSE.txt, or write to
# jos_AT_kirps_DOT_com or contact_AT_joopita_DOT_com.
#
################################################################################
# END OF COPYRIGHT / LICENSE, HERE COMES THE CODE ... #
################################################################################
use strict;
use ONO::IO;
use ONO::Lib::Basic;
use ONO::ToolBox::System;
use ONO::Cron::CronSync::ToolBox;
use ONO::Core::HostIO;
use ONO::Core::HostOS;
use ONO::DB;
use ONO::ToolBox::SendMail;
use ONO::ToolBox::Test;
use ONO::ToolBox::ONO;
use ONO::Lib::DateTime::ToolBox;
use ONO::ToolBox::Analytics;
use ONO::ToolBox::Security;
use ONO::Lib::Web::BotDetect;
###############################################################################
# ONO
###############################################################################
#: Operating System related services for ONO's CronSync service.
sub run {
my (
$self,
$CRON_DIR,
$job,
$data_ref,
$TYPE_ref,
$vars_ref,
$vpath,
$sec,$min,$hour,
$mday,$mon,$year,
$wday,$yday,
$timestamp,
$switches,
) = @_;
#: Execution logic.
my %data = %$data_ref;
my %TYPE = %$TYPE_ref;
my %vars = %$vars_ref;
my ($DATA,$MAIL);
if ($data{'type'} eq "os_security") {
if ($data{'os_security_system_users'}) {
if ($min%$data{'os_security_system_users'} == 0) {
my ($SITE,$DOMAIN,$FULLDOM) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN) {
my ($users,$USERS);
foreach my $who (ONO::Core::HostOS->who) {
if ($who) {
$users++;
$USERS .= "$who\n";
}
}
if ($users) {
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] LOGIN DETECTED ($users users, $hour:$min:$sec)",
qq~The following users are logged in:\n\n$USERS~,
);
}
}
}
}
if ($data{'os_security_apache_ip'}) {
if ($min%$data{'os_security_apache_ip'} == 0) {
my ($SITE,$DOMAIN,$FULLDOM) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN) {
$vars{'filter'} = "undef";
$vars{'ip_analytics_min_hits'} = $data{'os_security_apache_ip_hits'};
$vars{'ip_analytics_mode'} = "mail";
my %RULES;
foreach my $line (ONO::IO->list("etc/security/iptables_rules.txt")) {
if ($line =~ m~^(.*?):(.*?):~) {
$RULES{$1} = $2;
}
}
my ($LIST_ref) = ONO::ToolBox::Security->apache_log_ip_analytics(\%RULES,\%vars);
my @LIST = @$LIST_ref;
my $REP = $LIST[32];
if ($REP && $REP ne ONO::IO->load("var/tmp/security/os_security_apache_ip_mail.txt")) {
ONO::IO->mkpath("var/tmp/security");
ONO::IO->store("var/tmp/security/os_security_apache_ip_mail.txt",$REP);
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] SUSPICIOUS APACHE IP DETECTED",
qq~The following IPs have been detected:\n\nhttps://$FULLDOM/ono/admin/\?/info_security/\&filter=undef\n\n$REP~,
);
}
}
}
}
}
if ($data{'type'} eq "os_monitor") {
my ($errors,$REPORT) = ONO::ToolBox::Test->media_processors("H");
if ($errors) {
my ($SITE,$DOMAIN) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN) {
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] OS SERVICE MONITOR ERRORS ($errors)",
qq~The following errors have been reported:\n\n$REPORT~,
);
}
}
}
if ($data{'type'} eq "os_control") {
}
if ($data{'type'} eq "os_diskfree") {
my ($DISK,@disks);
foreach my $disk (ONO::IO->disks) {
$DISK .= $disk;
@disks = (@disks,$disk);
}
my ($REP,$MAX) = ONO::ToolBox::System->diskfree_display(\@disks,"Am");
$MAIL .= $REP;
if ($data{'os_diskfree_mail'} =~ /^(.*?)\@(.*?)\.(.*?)$/) {
my ($SITE,$DOMAIN) = ONO::Cron::CronSync::ToolBox->get_site();
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] Disk Usage Report [Max:$MAX\%]",
$MAIL,
);
}
ONO::IO->store("$CRON_DIR/diskfree.txt",$DISK);
}
if ($data{'type'} eq "os_chmod") {
if ($data{'os_chmod_logs'}) {
foreach my $file ('/var/log/apache2/access_log','/var/log/apache2/error_log','/var/log/apache2/access.log','/var/log/apache2/error.log') {
if (-e $file) {
$DATA .= ONO::IO->exec("chmod 644 $file");
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': chmod 644 $file",$switches);
}
}
}
if ($data{'os_chmod_www'} || $data{'os_chown_www'}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting chmod or chown process",$switches);
foreach my $dir (ONO::Core::HostIO->ls("/var/www")) {
if ($dir !~ /^\./ && $dir =~ /[A-Za-z0-9]/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': - Processing web '$dir' ...",$switches);
foreach my $subdir (ONO::Core::HostIO->ls("/var/www/$dir/html")) {
if ($subdir !~ /^\./ && $subdir =~ /[A-Za-z0-9]/ && $subdir !~ /^(etc)$/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': -- /var/www/$dir/html/$subdir ...",$switches);
if ($data{'os_chmod_www'}) {
ONO::Core::HostIO->chmodr("/var/www/$dir/",777);
}
if ($data{'os_chown_www'}) {
ONO::Core::HostIO->chowngrpr("/var/www/$dir/",$dir,$dir);
}
}
}
}
}
}
}
if ($data{'type'} eq "os_readconf") {
ONO::IO->exec("mkdir /etc/ono");
ONO::IO->exec("mkdir /etc/ono/readconf");
ONO::IO->exec("mkdir /etc/ono/readconf/etc");
unless (-e "/etc/ono/readconf/etc") {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR - /etc/ono/readconf/etc could not be created",$switches);
}
my %used;
foreach my $opt ('system','webserver','database','fileserver','mailserver') {
if ($data{"os_readconf_$opt"}) {
my @objs;
if ($opt eq "system") {
@objs = ('crontab','hostname','hosts','passwd');
}
if ($opt eq "database") {
@objs = ('mysql');
}
if ($opt eq "webserver") {
@objs = ('apache2','httpd');
}
if ($opt eq "fileserver") {
@objs = ('samba');
}
if ($opt eq "mailserver") {
@objs = ('postfix','dovecot','courier','mail','passwd','spamassassin','default/saslauthd','default/spamassassin');
}
foreach my $obj (@objs) {
if (-e "/etc/$obj" && !$used{$obj}) {
$used{$obj}++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Copying /etc/$obj to /etc/ono/readconf/etc/$obj on core system",$switches);
ONO::IO->exec("cp -r /etc/$obj /etc/ono/readconf/etc/");
ONO::IO->exec("chmod -R 755 /etc/ono/readconf/etc/$obj");
ONO::Core::HostIO->store("/etc/ono/readconf/etc/$obj.timestamp",$timestamp);
}
}
}
}
}
if ($data{'type'} eq "os_checkconf") {
my $COM = "cp -r /etc/apache2 $vpath/var/hostinfo/etc/";
ONO::IO->rmdir("var/hostinfo/etc");
ONO::IO->mkpath("var/hostinfo/etc");
ONO::IO->store("var/hostinfo/checkconf_report.txt","$COM\n");
ONO::IO->exec($COM);
ONO::IO->chmod("var/hostinfo/etc","777");
}
if ($data{'type'} eq "os_log_analyzer" && ONO::IO->sysload(50)) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting OS Log Analyzer",$switches);
ONO::Core::HostIO->cp("/var/log/apache2/access.log","$vpath/var/log/security/apache_access_log.txt");
ONO::Core::HostIO->cp("/var/log/apache2/access.log.1","$vpath/var/log/security/apache_access_log_previous.txt");
my (%STAT,%REFERERS,%AGENTS,%AUTH);
foreach my $opt ("/var/log/apache2/access.log","/var/log/apache2/access.log.1") {
if (ONO::Core::HostIO->exists($opt)) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Analyzing $opt",$switches);
my $WARNINGS;
foreach my $line (ONO::Core::HostIO->list($opt)) {
if ($line =~ m~(.*?) (.*?) \[(.*?)\] (.*?) "(.*?) (.*?) HTTP/(.*?)" (.*?) (.*?) "(.*?)" "(.*?)"~) {
my $domain = $1;
my $IP_32 = $2;
my $time = $3;
my $timestamp = $4;
my $method = $5;
my $url = $6;
my $referer = $10;
my $agent = $11;
$IP_32 =~ m~^(.*?)\.(.*?)\.(.*?)\.(.*?)$~;
my $IP_08 = $1;
my $IP_16 = "$1.$2";
my $IP_24 = "$1.$2.$3";
$STAT{"apache_ip_08_$IP_08"}++;
$STAT{"apache_ip_16_$IP_16"}++;
$STAT{"apache_ip_24_$IP_24"}++;
$STAT{"apache_ip_32_$IP_32"}++;
$AGENTS{"$IP_32=$agent"}++;
# $REFERERS{"$IP_32=$referer"}++;
}
# warn if evil bots, trackers or apps have been detected
if ($line =~ m~(copytrack|plaghunter|photoclaim|pixsy)~gi) {
ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Potentially evil access detected: $1",$switches);
$WARNINGS .= qq~$1 - $line~;
}
}
if ($WARNINGS) {
ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Warnings have been detected !",$switches);
my ($SITE,$DOMAIN,$FULLDOM) = ONO::Cron::CronSync::ToolBox->get_site();
if ($SITE && $DOMAIN && $min%30 == 0) {
ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Trying to send warnings to the tracker mail",$switches);
ONO::ToolBox::SendMail->sendmail(
"$DOMAIN <noreply\@$DOMAIN>",
"mail:tracker",
"[$SITE] WARNING - EVIL WEB ACCESS DETECTED",
qq~The following access.log entries look suspicious:\n\n$WARNINGS~,
);
}
}
}
}
foreach my $opt ("/var/log/auth.log","/var/log/auth.log.1") {
if (ONO::Core::HostIO->exists($opt)) {
foreach my $line (ONO::Core::HostIO->list($opt)) {
$line =~ s~(\n|\r|\t)~~g;
if ($line =~ / fail/gi) {
if ($line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+) ~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
if ($line =~ /root/ && $line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+) ~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
if ($line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+)$~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
if ($line =~ /root/ && $line =~ m~(=| )(\d+)\.(\d+)\.(\d+)\.(\d+)$~) {
if (!ONO::IO->is_local_ip("$2.$3.$4.$5")) {
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
$AUTH{"auth_ip_$2.$3.$4.$5"}++;
}
}
}
}
}
}
ONO::IO->mkpath("var/log/security");
my ($REPORT,$REPORT_AUTH,$REFS,$AGTS);
foreach my $key (reverse sort {$STAT{$a} <=> $STAT{$b}} keys %STAT) {
$REPORT .= qq~$key=$STAT{$key}\n~;
}
ONO::IO->store("var/log/security/report_ips.txt",$REPORT);
ONO::IO->store("var/log/security/report_ips_$year$mon$mday.txt",$REPORT);
foreach my $key (reverse sort {$REFERERS{$a} <=> $REFERERS{$b}} keys %REFERERS) {
$REFS .= qq~$key=$REFERERS{$key}\n~;
}
ONO::IO->store("var/log/security/report_referers.txt",$REFS);
foreach my $key (reverse sort {$AGENTS{$a} <=> $AGENTS{$b}} keys %AGENTS) {
$AGTS .= qq~$key=$AGENTS{$key}\n~;
}
ONO::IO->store("var/log/security/report_agents.txt",$AGTS);
foreach my $key (reverse sort {$AUTH{$a} <=> $AUTH{$b}} keys %AUTH) {
$REPORT_AUTH .= qq~$key=$AUTH{$key}\n~;
}
ONO::IO->store("var/log/security/report_ips_auth.txt",$REPORT_AUTH);
ONO::IO->store("var/log/security/report_ips_auth_$year$mon$mday.txt",$REPORT_AUTH);
}
if ($data{'type'} eq "os_firewall" && ONO::IO->sysload(50)) {
my (%FRIENDLY,$force_refresh);
foreach my $line (ONO::IO->list("etc/security/friendly_ips.txt")) {
$line =~ s~[^0-9\.]~~g;
$FRIENDLY{$line}++;
}
if ($data{'os_firewall_refresh'} > 0 && $data{'os_firewall_refresh'} eq $hour) {
if (!ONO::IO->exists("var/log/security/firewall_refresh_$year$mon$mday.txt")) {
ONO::IO->store("var/log/security/firewall_refresh_$year$mon$mday.txt","$hour:$min:$sec");
$force_refresh++;
}
}
if ($data{'os_firewall_auth_threshold'} > 0 || $data{'os_firewall_bad_script_threshold'} > 0 || $data{'os_firewall_bad_bot_threshold'} > 0) {
if (!ONO::IO->exists("var/log/security/firewall_tmp_flush_$year$mon$mday.txt")) {
ONO::IO->rm("etc/security/iptables_rules_tmp.txt");
ONO::IO->store("var/log/security/firewall_tmp_flush_$year$mon$mday.txt","$hour:$min:$sec");
}
my (%block,%used,%script,%bot);
if ($data{'os_firewall_auth_threshold'} > 0) {
foreach my $line (ONO::IO->list("var/log/security/report_ips_auth.txt")) {
$line =~ s~(\n|\r|\t)~~g;
if ($line =~ /^auth_ip_(.*?)=(.*?)$/) {
if (1+$2 > $data{'os_firewall_auth_threshold'} && !ONO::IO->is_local_ip($1)) {
$block{$1}++;
}
}
}
}
if ($min%5 == 0 && ($data{'os_firewall_bad_script_threshold'} > 0 || $data{'os_firewall_bad_bot_threshold'} > 0)) {
foreach my $line (ONO::IO->list("var/log/security/apache_access_log.txt")) {
if ($data{'os_firewall_bad_script_threshold'} > 0) {
if ($line =~ /\.php /i || $line =~ /wp-config\.php/i) {
if ($line =~ /^(.*?) (.*?)\.(.*?)\.(.*?)\.(.*?) \[/) {
my $ip = "$2.$3.$4.$5";
$script{$ip}++;
if ($script{$ip}+1 > $data{'os_firewall_bad_script_threshold'}) {
$block{$ip}++;
}
}
}
}
if ($data{'os_firewall_bad_bot_threshold'} > 0) {
# ONO::Lib::Web::BotDetect - first check if line even applies (faster...)
if (ONO::Lib::Web::BotDetect->evil_bot($line)) {
if ($line =~ /^(.*?) (.*?)\.(.*?)\.(.*?)\.(.*?) \[(.*?)\] (.*?) "(.*?)" (.*?) (.*?) "(.*?)" "(.*?)"/) {
my $ip = "$2.$3.$4.$5";
my $agent = $12;
# ONO::Lib::Web::BotDetect - then make sure it's really the user agent that's bad
if (ONO::Lib::Web::BotDetect->evil_bot($agent)) {
$bot{$ip}++;
if ($bot{$ip}+1 > $data{'os_firewall_bad_bot_threshold'}) {
$block{$ip}++;
}
}
}
}
}
}
}
foreach my $line (ONO::IO->list("etc/security/iptables_rules_tmp.txt")) {
my @lp = split(/\:/,$line);
if ($lp[1] eq "block") {
$used{$lp[0]}++;
}
}
foreach my $key (keys %block) {
if ($key =~ /^(.*?)\.(.*?)\.(.*?)\.(.*?)$/ && !$used{$key} && !$FRIENDLY{$key} && !ONO::IO->is_local_ip($key)) {
ONO::IO->append("etc/security/iptables_rules_tmp.txt","$key:block:\n");
`/sbin/iptables -A INPUT -s $key -j DROP`;
}
}
ONO::IO->chmod("etc/security/iptables_rules_tmp.txt","777");
}
if ($data{'os_firewall_auto_release_percent'} > 0) {
if (!ONO::IO->exists("var/log/security/firewall_release_$year$mon$mday.txt")) {
my ($DATA,$REL);
foreach my $line (ONO::IO->list("etc/security/iptables_rules.txt")) {
if ($line =~ m~(.*?)\.(.*?)\.(.*?)\.(.*?):block:~) {
if (ONO::Lib::Basic->random_percent($data{'os_firewall_auto_release_percent'})) {
$REL .= $line;
$force_refresh++;
} else {
$DATA .= $line;
}
} else {
$DATA .= $line;
}
}
ONO::IO->mv("etc/security/iptables_rules.txt","etc/security/iptables_rules_backup/$year$mon$mday$hour.txt");
ONO::IO->store("etc/security/iptables_rules.txt",$DATA);
ONO::IO->chmod("etc/security/iptables_rules.txt","777");
ONO::IO->store("var/log/security/firewall_release_$year$mon$mday.txt",$REL);
ONO::IO->chmod("var/log/security/firewall_release_$year$mon$mday.txt","777");
}
}
if (($min%5 == 0 && ONO::IO->exists("etc/security/iptables_rules_refresh.txt")) || $force_refresh) {
my ($DATA,$EXEC,%used,@BLOCK);
my $TRAP1 = substr(time(),0,4);
my $TRAP2 = $TRAP1 - 1;
`/sbin/iptables -F`;
`/sbin/iptables -N LOG_DROP`;
`/sbin/iptables -A LOG_DROP -j LOG --log-level warning --log-prefix "INPUT-DROP:"`;
`/sbin/iptables -A LOG_DROP -j DROP`;
foreach my $line (
ONO::IO->list("etc/security/iptables_rules_custom.txt"),
ONO::IO->list("etc/security/iptables_rules_tmp.txt"),
ONO::IO->list("etc/security/iptables_rules.txt"),
ONO::IO->list("iptables_rules_trap.$TRAP1.txt"),
ONO::IO->list("iptables_rules_trap.$TRAP2.txt"),
) {
my @lp = split(/\:/,$line);
if ($lp[1] eq "block" && !$FRIENDLY{$lp[0]} && !ONO::IO->is_local_ip($lp[0])) {
my @ips = split(/\./,$lp[0]);
my $num = @ips;
my ($ip,$range) = ("$lp[0].0.0.0",8);
if ($num == 2) {
($ip,$range) = ("$lp[0].0.0",16);
}
if ($num == 3) {
($ip,$range) = ("$lp[0].0",24);
}
if ($num == 4) {
($ip,$range) = ($lp[0],32);
}
if (!$used{"$ip/$range"} && !ONO::IO->is_local_ip($ip)) {
$used{"$ip/$range"}++;
@BLOCK = (@BLOCK,"$ip/$range");
}
}
}
my ($ipcount,$IPBLOCK);
foreach my $IP (@BLOCK) {
$ipcount++;
$IPBLOCK .= "$IP,";
if ($ipcount > 32) {
$ipcount = 0;
$IPBLOCK =~ s~\,$~~;
`/sbin/iptables -A INPUT -s $IPBLOCK -j LOG_DROP`;
$IPBLOCK = "";
}
}
if ($IPBLOCK) {
$IPBLOCK =~ s~\,$~~;
`/sbin/iptables -A INPUT -s $IPBLOCK -j LOG_DROP`;
}
ONO::IO->store("etc/security/iptables_rules_updated.txt","$mday/$mon/$year \@ $hour:$min:$sec");
# ONO::IO->store("etc/security/iptables_rules_blocked.txt","# Generated file, don't touch!\n\n$DATA");
# ONO::IO->store("etc/security/iptables_rules_exec.txt","# Generated file, don't touch!\n\n$EXEC");
my $IPTABLES;
foreach my $line (`/sbin/iptables -L -n -v`) {
if ($line =~ /DROP/) {
$line = ONO::Lib::Basic->remove_bad_spaces($line);
my @lp = split(/ /,$line);
$IPTABLES .= qq~$lp[7]\n~;
}
}
ONO::IO->store("var/log/security/iptables_rules_current.txt",$IPTABLES);
ONO::IO->rm("etc/security/iptables_rules_refresh.txt");
}
}
if ($data{'type'} eq "os_logfile" && ONO::IO->sysload(50)) {
my %conf = ONO::IO->confread("etc/logfileanalyzer.conf");
my $FILE = ONO::Lib::DateTime::ToolBox->yesterday($year,$mon,$mday);
my $REPORT = "Logfile dispatching on $mday/$mon/$year\n";
foreach my $opt ("/var/log/apache2/access.log","/var/log/apache2/access.log.1") {
if (ONO::Core::HostIO->exists($opt) && ($opt eq "/var/log/apache2/access.log" || (!ONO::IO->exists("var/log/apache/report_$FILE.txt") && $conf{'loglockhour'} == $hour))) {
$REPORT .= "Processing $opt...\n";
my %DATA;
foreach my $line (ONO::Core::HostIO->list($opt)) {
if ($line =~ m~(.*?) (.*?) \[(.*?)\] (.*?) "(GET|POST) (.*?) HTTP/1.1" (.*?) (.*?) "(.*?)" "(.*?)"~) {
my $domain = $1;
my $IP = $2;
my $time = $3;
my $timestamp = $4;
my $url = $6;
my $agent = $10;
$time =~ m~^(.*?)/(.*?)/(.*?):(.*?):(.*?):(.*?)~;
my $mm = ONO::Lib::DateTime::ToolBox->mmm2mm($2);
$time = "$3$mm$1";
if (!ONO::ToolBox::Analytics->exclude($url,$agent)) {
$DATA{$domain} .= qq~$domain $IP $time $timestamp "$url" "$agent"\n~;
}
$DATA{'*'} .= qq~$domain $IP $time $timestamp "$url" "$agent"\n~;
}
}
foreach my $key (keys %conf) {
if ($key =~ /^logfileexportrule/) {
$conf{$key} =~ m~^(.*?):(.*?)$~;
ONO::Core::HostIO->mkdir($2);
ONO::Core::HostIO->chmod($2,777);
if ($opt eq "/var/log/apache2/access.log") {
$REPORT .= "Store: $2/current.log\n";
ONO::Core::HostIO->store("$2/current.log",$DATA{$1});
ONO::Core::HostIO->chmod("$2/current.log",777);
}
if ($opt eq "/var/log/apache2/access.log.1") {
$REPORT .= "Store: $2/FILE.log\n";
ONO::Core::HostIO->store("$2/$FILE.log",$DATA{$1});
ONO::Core::HostIO->chmod("$2/$FILE.log",777);
ONO::IO->store("var/log/apache/report_$FILE.txt",$REPORT);
}
}
}
}
}
}
if ($data{'type'} eq "os_manage") {
my $write_report;
my $db = ONO::Cron::CronSync::ToolBox->connect();
if ($db) {
my $OS_TYPE = ONO::Core::HostOS->system();
my $IS_LINUX;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': OS is $OS_TYPE",$switches);
if ($OS_TYPE =~ /(Linux|Ubuntu|UNIX)/gi) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Does the OS allow to manage the system: YES",$switches);
$IS_LINUX++;
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Does the OS allow to manage the system: NO [ABORT]",$switches);
}
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database connected",$switches);
ONO::Core::HostIO->mkdir("/etc/ono/servermanager");
if ($data{"os_manage_system"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting system tasks...",$switches);
}
if ($data{"os_manage_webserver"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting webserver tasks...",$switches);
}
if ($data{"os_manage_database"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting database tasks...",$switches);
}
if ($data{"os_manage_fileserver"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting fileserver tasks...",$switches);
}
if ($data{"os_manage_mailserver"}) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Starting mailserver tasks...",$switches);
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/courier");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/default");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/dovecot");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/dovecot/conf.d");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/mail");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/postfix");
ONO::Core::HostIO->mkdir("/etc/ono/servermanager/mail/spamassassin");
my ($update_mailboxes,$update_domains,$update_addresses);
# mailboxes - create
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes","status = 'create'")) {
my @row = ONO::DB->readcols($line);
$update_mailboxes++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailbox '$row[2]/$row[3]' will be created...",$switches);
my $username = "ono-mail-$row[3]";
my $password = $row[4];
if ($password && !ONO::Core::HostIO->userget($username) && $IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Creating UNIX user '$row[3]' for Mailbox '$row[1]' ...",$switches);
if (!ONO::Core::HostIO->groupget("ono-mail")) {
ONO::Core::HostIO->groupadd("ono-mail");
}
ONO::Core::HostIO->useradd($username,$password,"ono-mail","/var/mail/$username","/bin/false");
ONO::Core::HostIO->mkdir("/var/mail/$username");
ONO::Core::HostIO->chown("/var/mail/$username",$username);
ONO::Core::HostIO->chgrp("/var/mail/$username","ono-mail");
ONO::Core::HostIO->chmod("/var/mail/$username","0750");
}
ONO::DB->command($db,"UPDATE ono_servermanager_mail_mailboxes SET status = '', password = '' WHERE id_10 = '$row[1]'");
}
# mailboxes - update
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes","status = 'update'")) {
my @row = ONO::DB->readcols($line);
$update_mailboxes++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailbox '$row[2]/$row[3]' will be updated...",$switches);
my $username = "ono-mail-$row[3]";
my $password = $row[4];
# add user if missing for some obscure reason...
if (!ONO::Core::HostIO->userget($username)) {
ONO::Core::HostIO->useradd($username,$password,"ono-mail","/var/mail/$username","/bin/false");
ONO::Core::HostIO->mkdir("/var/mail/$username");
ONO::Core::HostIO->chown("/var/mail/$username",$username);
ONO::Core::HostIO->chgrp("/var/mail/$username","ono-mail");
ONO::Core::HostIO->chmod("/var/mail/$username","0750");
}
# change user
if ($password && ONO::Core::HostIO->userget($username) && $IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Updating UNIX user '$username'...",$switches);
ONO::Core::HostIO->userpwd($username,$password);
}
ONO::DB->command($db,"UPDATE ono_servermanager_mail_mailboxes SET status = '' WHERE id_10 = '$row[1]'");
}
# mailboxes - delete
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes","status = 'delete'")) {
my @row = ONO::DB->readcols($line);
$update_mailboxes++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailbox '$row[2]/$row[3]' will be deleted...",$switches);
my $username = "ono-mail-$row[3]";
if (ONO::Core::HostIO->userget($username) && $IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Deleting UNIX user '$username'...",$switches);
ONO::Core::HostIO->userdel($username);
}
ONO::DB->command($db,"DELETE FROM ono_servermanager_mail_mailboxes WHERE id_10 = '$row[1]'");
}
# domains - create
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = 'create'")) {
my @row = ONO::DB->readcols($line);
$update_domains++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domain '$row[2]' will be created...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_domains SET status = '' WHERE id_10 = '$row[1]'");
}
# domains - update
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = 'update'")) {
my @row = ONO::DB->readcols($line);
$update_domains++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domain '$row[2]' will be updated...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_domains SET status = '' WHERE id_10 = '$row[1]'");
}
# domains - delete
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = 'delete'")) {
my @row = ONO::DB->readcols($line);
$update_domains++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domain '$row[2]' will be deleted...",$switches);
ONO::DB->command($db,"DELETE FROM ono_servermanager_mail_domains WHERE id_10 = '$row[1]'");
}
# addresses - create
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = 'create'")) {
my @row = ONO::DB->readcols($line);
$update_addresses++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail '$row[2]\@$row[3]' will be created...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_addresses SET status = '' WHERE id_10 = '$row[1]'");
}
# addresses - update
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = 'update'")) {
my @row = ONO::DB->readcols($line);
$update_addresses++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail '$row[2]\@$row[3]' will be updated...",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_addresses SET status = '' WHERE id_10 = '$row[1]'");
}
# addresses - delete
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = 'delete'")) {
my @row = ONO::DB->readcols($line);
$update_addresses++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail '$row[2]\@$row[3]' will be deleted...",$switches);
ONO::DB->command($db,"DELETE FROM ono_servermanager_mail_addresses WHERE id_10 = '$row[1]'");
}
# get data
my (%accounts,%domains);
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains")) {
my @row = ONO::DB->readcols($line);
$domains{$row[1]} = $row[2];
}
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_mailboxes")) {
my @row = ONO::DB->readcols($line);
$accounts{$row[1]} = $row[3];
}
# updates mailboxes
if ($update_mailboxes) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Mailboxes have been updated!",$switches);
}
# updates domains
if ($update_domains) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Domains have been updated!",$switches);
my $DATA;
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_domains","status = '' OR status IS NULL")) {
my @row = ONO::DB->readcols($line);
if ($row[2] =~ /^(.*?)\.(.*?)$/) {
$DATA .= "$row[2] ";
}
}
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/domains.txt",$DATA);
}
# updates addresses
if ($update_addresses) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': E-mail addresses have been updated!",$switches);
my ($ADDR,%addresses);
foreach my $line (ONO::DB->select($db,"ono_servermanager_mail_addresses","status = '' OR status IS NULL")) {
my @row = ONO::DB->readcols($line);
if ($row[4] =~ /[A-Za-z0-9]/) {
if ($row[4] =~ /^(.*?)\@(.*?)\.(.*?)$/) {
$addresses{"$row[2]\@$domains{$row[3]}"} .= "$row[4] ";
} else {
$addresses{"$row[2]\@$domains{$row[3]}"} .= "ono-mail-$accounts{$row[4]} ";
}
}
}
my $count = 0;
foreach my $key (sort keys %addresses) {
$count++;
$ADDR .= "$key $addresses{$key}\n";
}
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Putting $count e-mail addresses into the Postfix virtual file",$switches);
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/addresses.txt",$ADDR);
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/postfix/virtual",$ADDR);
}
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Database analysis and user updates completed...",$switches);
if ($update_mailboxes) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update mailboxes? YES",$switches);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update mailboxes? NO",$switches);
}
if ($update_domains) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update domains? YES",$switches);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update domains? NO",$switches);
}
if ($update_addresses) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update e-mail addresses? YES",$switches);
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Did we update e-mail addresses? NO",$switches);
}
# set this to 1 in order to always test updating the server config files:
my $force_test = 0;
# my $force_test = 1;
if ($force_test) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': WARNING! WE'RE RUNNING IN FORCE TEST MODE, NOT SUITED FOR PRODUCTION SYSTEMS !!!",$switches);
}
# generate server files
if ($force_test || $update_mailboxes || $update_domains || $update_addresses) {
$write_report++;
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Reading site setup...",$switches);
my %site = ONO::ToolBox::ONO->get();
my $HOSTNAME = $site{'domain'};
my $DOMAIN = $site{'domain'};
$DOMAIN =~ s~^(.*)\.(.*?)\.(.*?)$~$2\.$3~;
my $IP = $site{'ip'};
my $MAIL_DOMAIN_LIST = ONO::Core::HostIO->load("/etc/ono/servermanager/mail/domains.txt");
if ($HOSTNAME && $DOMAIN && $IP && $MAIL_DOMAIN_LIST) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Server Hostname is '$HOSTNAME'.",$switches);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Server Domain is '$DOMAIN'.",$switches);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Server IP is '$IP'.",$switches);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Generating server files...",$switches);
my @server_files = (
'dovecot/dovecot.conf',
'dovecot/conf.d/10-auth.conf',
'dovecot/conf.d/10-director.conf',
'dovecot/conf.d/10-logging.conf',
'dovecot/conf.d/10-mail.conf',
'dovecot/conf.d/10-master.conf',
'dovecot/conf.d/10-ssl.conf',
'dovecot/conf.d/10-tcpwrapper.conf',
'dovecot/conf.d/15-lda.conf',
'dovecot/conf.d/15-mailboxes.conf',
'dovecot/conf.d/20-imap.conf',
'dovecot/conf.d/90-acl.conf',
'dovecot/conf.d/90-plugin.conf',
'dovecot/conf.d/90-quota.conf',
'spamassassin/local.cf',
'mail/sendmail.conf',
'postfix/main.cf',
'postfix/master.cf',
'default/saslauthd',
'default/spamassassin',
);
foreach my $file (@server_files) {
my $DATA;
foreach my $line (ONO::IO->list("ono/sys/ONO/Resources/ServerConf/etc/$file")) {
if ($line !~ /^#/) {
$line =~ s~###MAIL_HOSTNAME###~$HOSTNAME~gi;
$line =~ s~###MAIL_DOMAIN###~$DOMAIN~gi;
$line =~ s~###MAIL_IP###~$IP~gi;
$line =~ s~###MAIL_SSL###~postfix~gi;
$line =~ s~###MAIL_DOMAINS###~$MAIL_DOMAIN_LIST~gi;
}
$DATA .= $line;
}
ONO::Core::HostIO->store("/etc/ono/servermanager/mail/$file",$DATA);
}
# write server configs to /etc (do this only on Linux, not on macOS)
if ($IS_LINUX) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': We're running on Linux [OK]",$switches);
my $valid_conf;
foreach my $line (ONO::Core::HostIO->list("/etc/ono/servermanager.conf")) {
if ($line =~ m/^writesystemfiles true/) {
$valid_conf++;
}
}
if ($valid_conf) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': /etc/ono/servermanager.conf looks good...",$switches);
# add the postfix virtual file to the list, as this file contains all e-mail addresses to be used
@server_files = (@server_files,'postfix/virtual');
# install all server configuration files (dovecot, sendmail, postfix, and saslauthd
foreach my $file (@server_files) {
if ($file !~ /^#/) {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Writing /etc/$file...",$switches);
ONO::Core::HostIO->cp("/etc/ono/servermanager/mail/$file","/etc/$file");
}
}
# reload / restart mail services, to make use the new settings will be used
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': reloading and restarting all mail server services...",$switches);
ONO::Core::HostIO->mailserver_reload();
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': /etc/ono/servermanager.conf not set up correctly [ABORT]",$switches);
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': This is NOT Linux [ABORT]",$switches);
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: /etc/site.conf didn't provide useful information [ABORT]",$switches);
}
}
# bounce mailbox management
#
# This requires /etc/mail/bounce.conf, including the following data:
#
# MailBoxes "/var/mail/test,/var/mail/test2,..." -> path(s) to the mailbox(es) where bounces are stored
# Sites "/var/www/html1,/var/www/html2,..." -> path(s) to the different sites on the server
#
# The target sites need to have their own /etc/mail/bouncetargets.txt, including the following data:
#
# match_string "/var/www/site/var/tmp/bouncedmails"
#
# match_string -> the sender e-mail that shall be tracked (noreply_AT_example_DOT_com, etc...)
# "/var/www/..." -> the "var/tmp/bouncedmails" directory of the target site, which MUST exist (will not be created)
my %bounce = ONO::IO->confread("etc/mail/bounce.conf");
if ($bounce{'mailboxes'} && $bounce{'sites'}) {
ONO::IO->mkdir("var/log/mail");
ONO::IO->chmod("var/log/mail",777);
ONO::IO->store("var/log/mail/bounce.txt","Mail Bounce is configured and running...\n");
my (%targets,@mails);
foreach my $mailbox (split(/,/,",$bounce{'mailboxes'},")) {
if ($mailbox =~ /[a-z]/) {
ONO::IO->append("var/log/mail/bounce.txt","Checking mailbox '$mailbox' ...\n");
foreach my $mail (ONO::Core::HostIO->ls("$mailbox")) {
if ($mail =~ /^1(.*?)\.(.*?)\.(.*)$/) {
if (!$targets{"-loaded"}) {
$targets{"-loaded"}++;
ONO::IO->append("var/log/mail/bounce.txt","Looking for target site data ...\n");
foreach my $site (split(/,/,",$bounce{'sites'},")) {
if ($site =~ /[a-z]/) {
ONO::IO->append("var/log/mail/bounce.txt","Loading target site '$site' data ...\n");
foreach my $line (ONO::Core::HostIO->list("$site/etc/mail/bouncetargets.txt")) {
$line =~ m~(.*?) \"(.*?)\"~;
$targets{$1} = $2;
@mails = (@mails,$1);
ONO::IO->append("var/log/mail/bounce.txt","Storing target: $1 leads to $2\n");
}
}
}
}
ONO::IO->append("var/log/mail/bounce.txt","Found: '$mail' ...\n");
my ($DATA,$target_found);
foreach my $line (ONO::Core::HostIO->list("$mailbox/$mail")) {
$DATA .= $line;
}
$DATA =~ s~(\n|\r|\t)~~g;
foreach my $test (@mails) {
if ($DATA =~ /$test/) {
ONO::IO->append("var/log/mail/bounce.txt","Matches: '$test', Move: '$mailbox/$mail', To: '$targets{$test}/' ...\n");
ONO::Core::HostIO->mv("$mailbox/$mail","$targets{$test}/$mail");
ONO::Core::HostIO->chmod("$targets{$test}",777);
ONO::Core::HostIO->chmod("$targets{$test}/$mail",777);
$target_found++;
}
}
if (!$target_found) {
ONO::IO->append("var/log/mail/bounce.txt","No target detected, mail '$mailbox/$mail' will be killed ...\n");
ONO::Core::HostIO->rm("$mailbox/$mail");
}
}
}
}
}
}
}
# trigger reload mailserver
if (ONO::IO->exists("$CRON_DIR/trigger/mailserver-$job.txt")) {
if ($IS_LINUX) {
ONO::Core::HostIO->mailserver_reload();
}
ONO::IO->store("$CRON_DIR/lastrun/restart-mailserver-$job.txt",$timestamp);
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Removing trigger file - $CRON_DIR/trigger/mailserver-$job.txt",$switches);
ONO::IO->rm("$CRON_DIR/trigger/mailserver-$job.txt");
}
# count disk usage (run about once an hour, on an irregular basis (1 to 100) per mailbox)
# set this to 1 in order to always test counting disk usage:
my $force_test = 0;
# my $force_test = 1;
foreach my $dir (ONO::Core::HostIO->ls("/var/mail")) {
if ($dir =~ /^ono-mail-(.*?)$/ && (int(rand(100)) == 50 || $force_test)) {
my $username = $1;
my $usage = ONO::Core::HostIO->usage("/var/mail/$dir");
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': Counting mailbox '$dir' size: ${usage}kb",$switches);
ONO::DB->command($db,"UPDATE ono_servermanager_mail_mailboxes SET quotaused = '$usage' WHERE username = '$username'");
}
}
} else {
$DATA .= ONO::Cron::CronSync::ToolBox->print("Job '$data{'title'}': ERROR: no database connection [ABORT]",$switches);
}
if ($write_report) {
ONO::Core::HostIO->store("/etc/ono/servermanager/report.txt",$DATA);
}
}
return ($DATA,$MAIL);
}
###############################################################################
# end of script
###############################################################################
1;
__END__