From 723148195f34b77c056848c17ac4b9bb3b552a7f Mon Sep 17 00:00:00 2001 From: arjen Date: Sat, 4 Jun 2005 07:15:16 +0000 Subject: [PATCH] Added pattern check on log entries with the service_pattern table. --- src/gcm_daemon/gcm_daemon.php | 269 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 263 insertions(+), 6 deletions(-) diff --git a/src/gcm_daemon/gcm_daemon.php b/src/gcm_daemon/gcm_daemon.php index 3ad8240..b75884c 100755 --- a/src/gcm_daemon/gcm_daemon.php +++ b/src/gcm_daemon/gcm_daemon.php @@ -20,7 +20,10 @@ Gnucomo-0.0.8: September 4th 2003 $Log: gcm_daemon.php,v $ - Revision 1.19 2004-01-10 20:04:12 arjen + Revision 1.20 2005-06-04 07:15:16 arjen + Added pattern check on log entries with the service_pattern table. + + Revision 1.19 2004/01/10 20:04:12 arjen Send email about open notifications to an object's users. Revision 1.18 2003/12/03 08:07:21 arjen @@ -66,11 +69,13 @@ */ -// $Id: gcm_daemon.php,v 1.19 2004-01-10 20:04:12 arjen Exp $ +// $Id: gcm_daemon.php,v 1.20 2005-06-04 07:15:16 arjen Exp $ ini_set('include_path', '.:./classes:../phpclasses'); ini_set('html_errors', 'false'); +define("BATCHSIZE", 200); + //Tell the log that we're up. define_syslog_variables(); @@ -83,7 +88,7 @@ require_once "gnucomo.process_log.php"; $project_name = "gnucomo"; // name of the entire project $app_name = "gcm_daemon"; // name of the application running $developrelease = "FALSE"; // Indicates if special debug settings are needed -$db_version = 44; // The db_version indicates what the level of +$db_version = 48; // The db_version indicates what the level of // the database should be. If the database is // old an update will be generated. $gcmd_version = 5; // This value indicates the active version of @@ -206,6 +211,7 @@ do //At this place we start processing new log-lines + echo "Processing logs...\n"; process_log (); service_check(); find_notifications(); @@ -256,9 +262,11 @@ function process_log () $last_log = $dbms->db_result_row[0]; } + echo "Last processed logid = $last_log \n"; + //Query the log-table $local_sql = "SELECT * FROM log WHERE logid > CAST(".$last_log." AS BIGINT) - ORDER BY logid"; + ORDER BY logid LIMIT " . BATCHSIZE; $dbms->query($local_sql); //Update the log-statistics in the object-table @@ -352,9 +360,16 @@ function process_log () $local_counter++; } + // Check for spam and other abuses. + + // abuse_check($last_log); + + match_log_patterns($last_log); + // Register that the logrecords have been processed. + $local_upper_row--; $local_sql = "UPDATE db_value SET setting_value = '" - .$local_log_id."' where setting = 'log_processing'"; + . $local_upper_row ."' where setting = 'log_processing'"; $dbms->query($local_sql); @@ -455,6 +470,223 @@ function GatherStatistics($objectid) UpdateStatistic($objectid, 'logs', $cnt->count); } +// Check for spam and other abuses in the log_adv tables. + +function abuse_check($logstart) +{ +return; // This function is obsolete + global $dbms; + + // notification: 'abuses exceeded'. + + $noqueue_res = $dbms->query("select logid, source_ip from log_adv_daemon_email + where event='NOQUEUE' and logid > " . $logstart); + echo "NOQUEUE abuses:\n\n"; + for ($row = 0; $row < $dbms->num_rows($noqueue_res); $row++) + { + $noqueue = $dbms->fetch_object($noqueue_res, $row); + if ($noqueue->source_ip != '') + { + $obj = $dbms->fetch_object( + $dbms->query("SELECT objectid FROM log WHERE logid = '" . $noqueue->logid . "'"),0); + record_abuse(0, $obj->objectid, $noqueue->source_ip, 2); + + // TODO: Create notification + } + } + $dbms->Free($noqueue_res); + + $noqueue_res = $dbms->query("select logid, source_ip, relay from log_adv_daemon_email + where event='SPAM' and logid > " . $logstart); + echo "SPAM abuses:\n\n"; + for ($row = 0; $row < $dbms->num_rows($noqueue_res); $row++) + { + $noqueue = $dbms->fetch_object($noqueue_res, $row); + $source = $noqueue->source_ip; + if ($source == '') + { + $source = $noqueue->relay; + } + if ($source != '') + { + $obj = $dbms->fetch_object( + $dbms->query("SELECT objectid FROM log WHERE logid = '" . $noqueue->logid . "'"),0); + record_abuse(0, $obj->objectid, $source, 1); + + // TODO: Create notification + } + } + $dbms->Free($noqueue_res); + + echo "HTTP abuses:\n\n"; + $abuse_res = $dbms->query("select logid, objectid, rawdata from log + where servicecode='httpd' and logid > " . $logstart); + for ($row = 0; $row < $dbms->num_rows($abuse_res); $row++) + { + $source = ''; + $abuse = $dbms->fetch_object($abuse_res, $row); + if (ereg("\[error\] \[client ([0-9.]+)\] request failed: URI too long", $abuse->rawdata, $parts)) + { + echo $abuse->rawdata . "\n"; + echo "Abuse on object " . $abuse->objectid . " from IP address " . $parts[1] . "\n"; + $source = $parts[1]; + } + if (ereg("\[error\] \[client ([0-9.]+)\] File does not exist: .+/MSADC", + $abuse->rawdata, $parts)) + { + echo $abuse->rawdata . "\n"; + echo "Abuse on object " . $abuse->objectid . " from IP address " . $parts[1] . "\n"; + $source = $parts[1]; + } + if ($source != '') + { + record_abuse(0, $abuse->objectid, $source, 2); + + // TODO: Create notification + } + } + $dbms->Free($abuse_res); +} + +function match_log_patterns($logstart) +{ + global $dbms; + + $notifications = array(); + + // notification: 'abuses exceeded'. + + $noqueue_res = $dbms->query("select logid, objectid, servicecode, rawdata from log where logid > " . $logstart + . " order by logid limit " . BATCHSIZE); + for ($row = 0; $row < $dbms->num_rows($noqueue_res); $row++) + { + $logentry = $dbms->fetch_object($noqueue_res, $row); + echo "\n----------\n" . $logentry->rawdata . "\n----------\n"; + $service = $logentry->servicecode; + $pattern_res = $dbms->query("select * from service_pattern where service='$service' + OR service='ANY' order by rank"); + + $match_found = false; + for ($patnr = 0; !$match_found && $patnr < $dbms->num_rows($pattern_res); $patnr++) + { + $srv_pat = $dbms->fetch_object($pattern_res, $patnr); + if (ereg($srv_pat->pattern, $logentry->rawdata, $matches)) + { + // Scan the argument for '$n' expressions and expand + + $srv_pat->argument = expand_arguments($srv_pat->argument, $matches); + echo " " . $srv_pat->pattern . " matches.\n"; + echo " Matched string: " . $matches[0] . "\n"; + echo " Action = " . $srv_pat->action . "(" . $srv_pat->argument . ")\n\n"; + $match_found = true; + + switch ($srv_pat->action) + { + case "ignore": + break; + + case "notify": + $notif = $srv_pat->argument; + if (!isset($notifications[$logentry->objectid][$notif])) + { + echo "Creating notification $notif for object " . $logentry->objectid . ".\n"; + $remark = "Notification generated from Gnucomo pattern match."; + $notifications[$logentry->objectid][$notif] = + $dbms->new_notification($logentry->objectid, $notif, $remark); + } + if (isset($notifications[$logentry->objectid][$notif])) + { + echo "Notification $notif for object " . $logentry->objectid . " already created.\n"; + $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('"; + $insertion .= $notifications[$logentry->objectid][$notif] . "', '"; + $insertion .= $logentry->logid . "')"; + $dbms->query($insertion); + } + break; + + case "abuse": + if (record_abuse($logentry->logid, $logentry->objectid, $srv_pat->argument, 1) >= 6) + { + $source_ip = $srv_pat->argument; + $notif = 'abuses exceeded'; + if (!isset($notifications[$logentry->objectid][$notif][$source_ip])) + { + echo "Creating notification $notif for object " . $logentry->objectid . ".\n"; + $remark = "Abuses from IP address $source_ip exceeded the limit."; + $notifications[$logentry->objectid][$notif][$source_ip] = + $dbms->new_notification($logentry->objectid, $notif, $remark); + } + if (isset($notifications[$logentry->objectid][$notif][$source_ip])) + { + echo "Notification $notif for object " . $logentry->objectid . " already created.\n"; + $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('"; + $insertion .= $notifications[$logentry->objectid][$notif][$source_ip] . "', '"; + $insertion .= $logentry->logid . "')"; + $dbms->query($insertion); + } + } + break; + default: + echo "Error: unrecognized action in service pattern: " . $srv_pat->action . "\n"; + break; + } + } + else + { + echo " " . $srv_pat->pattern . " does not match.\n"; + } + } + + } +} + +/* + * Some IP address abused us. Record the event. + * Return the number of abuse points recorded so far for the address + */ + +function record_abuse($logid, $objectid, $sourceip, $points) +{ + global $dbms; + + $abuse_points = $points; + + $ipaddress = gethostbyname($sourceip); + echo " IP address for $sourceip is $ipaddress.\n"; + $sourceip = $ipaddress; + + $abres = $dbms->query("SELECT * FROM object_abuse WHERE objectid='$objectid' AND source='$sourceip'"); + + if (pg_numrows($abres) == 0) + { + echo "$sourceip is new.\n"; + $dbms->query("INSERT INTO object_abuse VALUES ('$objectid', '$sourceip', '$points')"); + } + else + { + $abuse = $dbms->fetch_object($abres, 0); + if ($abuse->status == '' || $abuse->status == 'dropped') + { + $abuse_points = $abuse->nr_abuses + $points; + echo $sourceip . " will get " . $abuse_points . " abuse points, "; + echo "Status was " . $abuse->status . "\n"; + $dbms->query("UPDATE object_abuse SET nr_abuses='$abuse_points'" . + " WHERE objectid='$objectid' AND source='$sourceip'"); + + $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')"); + if ($abuse_points >= 6) + { + echo " BLOCK IP adrress $sourceip on the firewall.\n"; + $dbms->query("UPDATE object_abuse SET status='dropped'" . + " WHERE objectid='$objectid' AND source='$sourceip'"); + } + } + } + + return $abuse_points; +} + + /* * Service_check - Check the log entries if there are any unknown * services. @@ -486,7 +718,7 @@ function service_check() // Query the log-table $qry = "SELECT logid, objectid, servicecode FROM log - WHERE logid > CAST(".$last_log." AS BIGINT) ORDER BY logid"; + WHERE logid > CAST(".$last_log." AS BIGINT) ORDER BY logid LIMIT " . BATCHSIZE; $log_res = $dbms->query($qry); //$log_res = $dbms->query("SELECT logid, objectid, servicecode,rawdata FROM log"); @@ -591,6 +823,7 @@ function mail_notifications () $issue = pg_fetch_object($dbms->query("SELECT description FROM type_of_issue WHERE type_of_issueid='" . $notification->type_of_issueid . "'"), 0); + echo "Mailing Notification for object id " . $notification->objectid . "\n"; $object = pg_fetch_object($dbms->query("SELECT objectname FROM object WHERE objectid='" . $notification->objectid ."'"), 0); @@ -610,5 +843,29 @@ function mail_notifications () } } +/* + * The 'command' may contain positional parameters such as '$1' and '$3', + * just like the shell. These parameters are replaced by content from + * the 'args' array. + */ + +function expand_arguments($command, $args) +{ + while (ereg('\$([0-9]+)', $command, $match)) + { + $index = $match[1]; + if ($index >= count($args)) + { + echo "Error: Argument $index not found for $command.\n"; + $command = ereg_replace('\$' . $index, "", $command); + } + else + { + $command = ereg_replace('\$' . $index, $args[$index], $command); + } + } + return $command; +} + ?> -- 2.11.0