From 1329f1ca9285415eff8fa1802798ebda5764901d Mon Sep 17 00:00:00 2001 From: arjen Date: Thu, 11 Jan 2007 13:47:35 +0000 Subject: [PATCH] Log_adv and derived tables removed. Create notifications from log entries with pattern matching. --- src/database/create.sql | 122 ++------------------ src/gcm_daemon/classes/gnucomo_db_version.php | 25 +++++ src/gcm_daemon/gcm_daemon.php | 153 +++++++++++++++++++++----- 3 files changed, 160 insertions(+), 140 deletions(-) diff --git a/src/database/create.sql b/src/database/create.sql index 99f9561..a17028a 100644 --- a/src/database/create.sql +++ b/src/database/create.sql @@ -12,7 +12,11 @@ -- DBA create the database and give access permissions. -- -- $Log: create.sql,v $ --- Revision 1.19 2005-06-04 07:09:47 arjen +-- Revision 1.20 2007-01-11 13:47:35 arjen +-- Log_adv and derived tables removed. +-- Create notifications from log entries with pattern matching. +-- +-- Revision 1.19 2005/06/04 07:09:47 arjen -- - Fixed field declaration for PostgreSQL 7.4 -- - New tables: log_abuse, object_abuse and service_pattern -- - Added general service patterns in the service 'ANY' @@ -230,111 +234,6 @@ CREATE TABLE log_abuse -- -- -CREATE SEQUENCE "log_advid_seq"; - - -CREATE TABLE "log_adv" -( - "log_advid" bigint DEFAULT - nextval('"log_advid_seq"'::text) NOT NULL, - "logid" bigint NOT NULL, - "detailed_table" text -); - - -CREATE INDEX log_adv_logid ON log_adv USING btree (logid); - -CREATE UNIQUE INDEX log_adv_log_advid ON log_adv USING btree (log_advid); - -SELECT setval ('"log_advid_seq"', 1, false); - -CREATE TABLE log_adv_kernel_network -( - device_in text, - device_out text, - hw_address text, - source_ip INET, - destination_ip INET, - packet_length bigint, - tos_bit text, - prec_bit text, - ttl int, - header_id bigint, - source_port int, - destination_port int, - body_length int, - protocol text, - body_len int, - window text, - urgp int, - syn boolean DEFAULT false, - type int, - code int, - sequence_number int, - res text, - rst boolean, - df boolean -) INHERITS (log_adv); - -CREATE TABLE log_adv_daemon -( - service TEXT, - event TEXT -) INHERITS (log_adv); - -CREATE INDEX log_adv_daemon_service ON log_adv_daemon (service); - -CREATE TABLE log_adv_daemon_email -( - source_ip INET, - destination_ip INET, - internal_messageid TEXT, - external_messageid TEXT, - to_email TEXT, - from_email TEXT, - delay interval, - xdelay interval, - mailer TEXT, - status TEXT, - pid INT, - size INT, - pri INT, - relay TEXT, - status_details TEXT, - dsn TEXT -) INHERITS (log_adv_daemon); - -CREATE INDEX log_adv_daemon_email_s_ip ON log_adv_daemon_email (source_ip); -CREATE INDEX log_adv_daemon_email_d_ip ON log_adv_daemon_email (destination_ip); -CREATE INDEX log_adv_daemon_email_to ON log_adv_daemon_email (to_email); -CREATE INDEX log_adv_daemon_email_from ON log_adv_daemon_email (from_email); -CREATE INDEX log_adv_email_status ON log_adv_daemon_email (status); - - -/* -CREATE TRUSTED PROCEDURAL LANGUAGE 'plpgsql' HANDLER "plpgsql_call_handler" LANCOMPILER ''; -CREATE FUNCTION "funct_processlog" () RETURNS opaque AS ' --- Initial date: September 18th 2002 --- Update: November 13th 2002 --- Author: Brenno J.S.A.A.F. de Winter --- Abstract: This routine sets a flag -DECLARE - var_value VARCHAR; - var_setting VARCHAR; - -BEGIN - - UPDATE db_value SET setting_value = ''TRUE'' WHERE setting = ''log_flag''; - RETURN NULL; -END; - ' LANGUAGE 'plpgsql'; - - -CREATE TRIGGER "log_insert" AFTER INSERT ON "log" FOR EACH ROW EXECUTE PROCEDURE "funct_processlog" (); -*/ --- --- - CREATE TABLE "log_notification" ( "notificationid" bigint, @@ -506,6 +405,7 @@ CREATE TABLE "object_abuse" source inet, nr_abuses integer, status text, + last_change timestamp, primary key (objectid, source) ); @@ -851,10 +751,6 @@ GRANT SELECT ON action_user TO GROUP view, GROUP ops, GROUP admin; GRANT SELECT ON db_value TO GROUP view, GROUP ops, GROUP admin; GRANT SELECT ON history TO GROUP view, GROUP ops, GROUP admin; GRANT SELECT ON log TO GROUP view, GROUP ops, GROUP admin; -GRANT SELECT ON log_adv TO GROUP view, GROUP ops, GROUP admin; -GRANT SELECT ON log_adv_daemon TO GROUP view, GROUP ops, GROUP admin; -GRANT SELECT ON log_adv_daemon_email TO GROUP view, GROUP ops, GROUP admin; -GRANT SELECT ON log_adv_kernel_network TO GROUP view, GROUP ops, GROUP admin; GRANT SELECT ON log_notification TO GROUP view, GROUP ops, GROUP admin; GRANT SELECT ON notification TO GROUP view, GROUP ops, GROUP admin; GRANT SELECT ON object TO GROUP view, GROUP ops, GROUP admin; @@ -876,6 +772,7 @@ GRANT SELECT ON type_of_issue TO GROUP view, GROUP ops, GROUP admin; GRANT SELECT ON usr TO GROUP view, GROUP ops, GROUP admin; GRANT INSERT ON action_user TO GROUP ops, GROUP admin; +GRANT UPDATE ON action_user_actionstepid_seq TO GROUP ops, GROUP admin; GRANT UPDATE ON notification TO GROUP ops, GROUP admin; GRANT INSERT ON object TO GROUP admin; GRANT UPDATE ON object TO GROUP ops, GROUP admin; @@ -886,3 +783,8 @@ GRANT DELETE ON object_abuse TO GROUP admin; GRANT UPDATE ON usr TO GROUP view, GROUP ops, GROUP admin; GRANT INSERT ON usr TO GROUP admin; GRANT DELETE ON usr TO GROUP admin; +GRANT SELECT ON service_pattern TO GROUP view, GROUP ops, GROUP admin; +GRANT INSERT ON service_pattern TO GROUP admin; +GRANT UPDATE ON service_pattern TO GROUP ops, GROUP admin; +GRANT DELETE ON service_pattern TO GROUP admin; +GRANT SELECT ON log_abuse TO GROUP view, GROUP ops, GROUP admin; diff --git a/src/gcm_daemon/classes/gnucomo_db_version.php b/src/gcm_daemon/classes/gnucomo_db_version.php index 9ce5cfb..c927b0b 100644 --- a/src/gcm_daemon/classes/gnucomo_db_version.php +++ b/src/gcm_daemon/classes/gnucomo_db_version.php @@ -612,9 +612,34 @@ case 43: $local_sql .= " VALUES ('Warning detected',3,'A Warning is reported in the log', true) "; $dbms->query($local_sql); + case 48: + + $dbms->query("GRANT SELECT ON service_pattern TO GROUP view, GROUP ops, GROUP admin"); + $dbms->query("GRANT INSERT ON service_pattern TO GROUP admin"); + $dbms->query("GRANT UPDATE ON service_pattern TO GROUP ops, GROUP admin"); + $dbms->query("GRANT DELETE ON service_pattern TO GROUP admin"); + $dbms->query("GRANT SELECT ON log_abuse TO GROUP view, GROUP ops, GROUP admin"); + + case 49: + $local_sql = "ALTER TABLE object_abuse ADD COLUMN last_change timestamp"; + $dbms->query($local_sql); + + case 50: + + $dbms->query("DROP SEQUENCE log_advid_seq"); + $dbms->query("DROP TABLE log_adv_kernel_network"); + $dbms->query("DROP TABLE log_adv_daemon_email"); + $dbms->query("DROP TABLE log_adv_daemon"); + $dbms->query("DROP TABLE log_adv"); //These columns have to be removed when a new version of PGSQL has become mainstream that supportsa DROP COLUMN /* +TODO: + +GRANT UPDATE ON action_user_actionstepid_seq TO GROUP ops, GROUP admin; + +*/ +/* $local_sql = "ALTER TABLE log DROP COLUMN recognized"; $dbms->query($local_sql); diff --git a/src/gcm_daemon/gcm_daemon.php b/src/gcm_daemon/gcm_daemon.php index b75884c..7003d95 100755 --- a/src/gcm_daemon/gcm_daemon.php +++ b/src/gcm_daemon/gcm_daemon.php @@ -20,7 +20,11 @@ Gnucomo-0.0.8: September 4th 2003 $Log: gcm_daemon.php,v $ - Revision 1.20 2005-06-04 07:15:16 arjen + Revision 1.21 2007-01-11 13:47:41 arjen + Log_adv and derived tables removed. + Create notifications from log entries with pattern matching. + + Revision 1.20 2005/06/04 07:15:16 arjen Added pattern check on log entries with the service_pattern table. Revision 1.19 2004/01/10 20:04:12 arjen @@ -69,12 +73,12 @@ */ -// $Id: gcm_daemon.php,v 1.20 2005-06-04 07:15:16 arjen Exp $ +// $Id: gcm_daemon.php,v 1.21 2007-01-11 13:47:41 arjen Exp $ ini_set('include_path', '.:./classes:../phpclasses'); ini_set('html_errors', 'false'); -define("BATCHSIZE", 200); +define("BATCHSIZE", 2000); //Tell the log that we're up. define_syslog_variables(); @@ -85,10 +89,11 @@ require_once "gnucomo.process_log.php"; // Set the standard variables // +$purge_date =""; // Purge log entries until this date. Default: no purging $project_name = "gnucomo"; // name of the entire project $app_name = "gcm_daemon"; // name of the application running $developrelease = "FALSE"; // Indicates if special debug settings are needed -$db_version = 48; // The db_version indicates what the level of +$db_version = 51; // The db_version indicates what the level of // the database should be. If the database is // old an update will be generated. $gcmd_version = 5; // This value indicates the active version of @@ -111,8 +116,13 @@ for ($argi = 1; $argi < $argc; $argi++) $project_name = $argv[$argi]; break; + case "-p": + $argi++; + $purge_date = $argv[$argi]; + break; + default: - echo "Usage: gcm_daemon [-c configname]\n"; + echo "Usage: gcm_daemon [-c configname] [-p purgedate]\n"; exit(); break; } @@ -209,13 +219,16 @@ if ($dbms->fetch_row() == "TRUE") do { - //At this place we start processing new log-lines + if ($purge_date != "") + { + purge_old_logs($purge_date); + } echo "Processing logs...\n"; process_log (); service_check(); find_notifications(); - mail_notifications(); + //mail_notifications(); // Gather the statistics for each object @@ -235,6 +248,53 @@ do syslog (LOG_INFO, "gcm_daemon ended nicely"); +function purge_old_logs($purge_date) +{ + global $dbms; + + /* + * Make a temporary table with the logids of the old log entries + * We don't want to repeat a selection on the large log table itself. + */ + + echo "Purging log entries before $purge_date\n"; + + $dbms->query("CREATE TABLE gcm_deamon_old_log AS SELECT logid FROM log WHERE logid < $purge_date"); + $dbms->query("SELECT logid FROM gcm_deamon_old_log"); + echo $dbms->num_rows() . " log entries found.\n"; + $r = $dbms->query("select notificationid from log_notification where logid in + (select logid from gcm_deamon_old_log) group by notificationid"); + echo "Notifications that may be affected:\n"; + $notifications = array(); + for ($i=0; $i < $dbms->num_rows(); $i++) + { + $notif = $dbms->fetch_object($r, $i); + $notifications[] = $notif->notificationid; + echo $notif->notificationid . "\n"; + } + $dbms->query("delete from log_notification where logid in + (select logid from gcm_deamon_old_log)"); + + // Clean up any notifications that have no more logs left + foreach ($notifications as $notif) + { + $c = $dbms->fetch_object($dbms->query("select count(*) from log_notification where notificationid=$notif"), 0); + echo "Notification $notif has " . $c->count . " log entries left.\n"; + if ($c->count == 0) + { + echo "Cleaning up notification $notif.\n"; + $dbms->query("delete from action_user where notificationid=$notif"); + $dbms->query("delete from notification where notificationid=$notif"); + } + } + + $dbms->query("delete from log where logid in + (select logid from gcm_deamon_old_log)"); + + $dbms->query("drop table gcm_deamon_old_log"); +} + + function process_log () { @@ -265,8 +325,9 @@ function process_log () echo "Last processed logid = $last_log \n"; //Query the log-table + $log_limit = $last_log + BATCHSIZE; $local_sql = "SELECT * FROM log WHERE logid > CAST(".$last_log." AS BIGINT) - ORDER BY logid LIMIT " . BATCHSIZE; + AND logid <= $log_limit ORDER BY logid"; $dbms->query($local_sql); //Update the log-statistics in the object-table @@ -306,6 +367,7 @@ function process_log () $local_object_os = ""; $local_object_os_version = ""; +/* while ($local_counter < $dbms->num_rows()) { @@ -359,6 +421,7 @@ function process_log () } $local_counter++; } +*/ // Check for spam and other abuses. @@ -554,14 +617,14 @@ function match_log_patterns($logstart) $notifications = array(); - // notification: 'abuses exceeded'. - - $noqueue_res = $dbms->query("select logid, objectid, servicecode, rawdata from log where logid > " . $logstart - . " order by logid limit " . BATCHSIZE); + $log_limit = $logstart + BATCHSIZE; + $noqueue_res = $dbms->query("select logid, objectid, servicecode, rawdata from log + where logid > $logstart and logid <= $log_limit + order by logid"); for ($row = 0; $row < $dbms->num_rows($noqueue_res); $row++) { $logentry = $dbms->fetch_object($noqueue_res, $row); - echo "\n----------\n" . $logentry->rawdata . "\n----------\n"; + //echo "\n----------\n" . $logentry->rawdata . "\n----------\n"; $service = $logentry->servicecode; $pattern_res = $dbms->query("select * from service_pattern where service='$service' OR service='ANY' order by rank"); @@ -575,9 +638,9 @@ function match_log_patterns($logstart) // Scan the argument for '$n' expressions and expand $srv_pat->argument = expand_arguments($srv_pat->argument, $matches); - echo " " . $srv_pat->pattern . " matches.\n"; - echo " Matched string: " . $matches[0] . "\n"; - echo " Action = " . $srv_pat->action . "(" . $srv_pat->argument . ")\n\n"; + //echo " " . $srv_pat->pattern . " matches.\n"; + //echo " Matched string: " . $matches[0] . "\n"; + //echo " Action = " . $srv_pat->action . "(" . $srv_pat->argument . ")\n\n"; $match_found = true; switch ($srv_pat->action) @@ -589,14 +652,14 @@ function match_log_patterns($logstart) $notif = $srv_pat->argument; if (!isset($notifications[$logentry->objectid][$notif])) { - echo "Creating notification $notif for object " . $logentry->objectid . ".\n"; + //echo "Creating notification $notif for object " . $logentry->objectid . ".\n"; $remark = "Notification generated from Gnucomo pattern match."; $notifications[$logentry->objectid][$notif] = $dbms->new_notification($logentry->objectid, $notif, $remark); } if (isset($notifications[$logentry->objectid][$notif])) { - echo "Notification $notif for object " . $logentry->objectid . " already created.\n"; + //echo "Notification $notif for object " . $logentry->objectid . " already created.\n"; $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('"; $insertion .= $notifications[$logentry->objectid][$notif] . "', '"; $insertion .= $logentry->logid . "')"; @@ -613,8 +676,25 @@ function match_log_patterns($logstart) { echo "Creating notification $notif for object " . $logentry->objectid . ".\n"; $remark = "Abuses from IP address $source_ip exceeded the limit."; - $notifications[$logentry->objectid][$notif][$source_ip] = - $dbms->new_notification($logentry->objectid, $notif, $remark); + $notifid = $dbms->new_notification($logentry->objectid, $notif, $remark); + $notifications[$logentry->objectid][$notif][$source_ip] = $notifid; + + // Add log entries from previously detected abuses + + echo " Add log entries from previously detected abuses"; + $abuses = $dbms->query("SELECT logid FROM log_abuse WHERE objectid = '" . + $logentry->objectid . "' AND source = '$source_ip'"); + for ($abusenr = 0; $abusenr < $dbms->num_rows($abuses); $abusenr++) + { + $log_abuse = $dbms->fetch_object($abuses, $abusenr); + if ($log_abuse->logid != $logentry->logid) + { + $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('"; + $insertion .= $notifid . "', '"; + $insertion .= $log_abuse->logid . "')"; + $dbms->query($insertion); + } + } } if (isset($notifications[$logentry->objectid][$notif][$source_ip])) { @@ -626,6 +706,9 @@ function match_log_patterns($logstart) } } break; + case "forgive": + record_abuse($logentry->logid, $logentry->objectid, $srv_pat->argument, -4); + break; default: echo "Error: unrecognized action in service pattern: " . $srv_pat->action . "\n"; break; @@ -633,7 +716,7 @@ function match_log_patterns($logstart) } else { - echo " " . $srv_pat->pattern . " does not match.\n"; + // echo " " . $srv_pat->pattern . " does not match.\n"; } } @@ -652,31 +735,38 @@ function record_abuse($logid, $objectid, $sourceip, $points) $abuse_points = $points; $ipaddress = gethostbyname($sourceip); - echo " IP address for $sourceip is $ipaddress.\n"; + //echo " IP address for $sourceip is $ipaddress.\n"; $sourceip = $ipaddress; $abres = $dbms->query("SELECT * FROM object_abuse WHERE objectid='$objectid' AND source='$sourceip'"); - if (pg_numrows($abres) == 0) + if (pg_numrows($abres) == 0 && $points > 0) { - echo "$sourceip is new.\n"; + //echo "$sourceip is new.\n"; $dbms->query("INSERT INTO object_abuse VALUES ('$objectid', '$sourceip', '$points')"); } - else + else if (pg_numrows($abres) != 0) { $abuse = $dbms->fetch_object($abres, 0); if ($abuse->status == '' || $abuse->status == 'dropped') { $abuse_points = $abuse->nr_abuses + $points; - echo $sourceip . " will get " . $abuse_points . " abuse points, "; - echo "Status was " . $abuse->status . "\n"; + if ($abuse_points < 0) + { + $abuse_points = 0; + } + //echo $sourceip . " will get " . $abuse_points . " abuse points, "; + //echo "Status was " . $abuse->status . "\n"; $dbms->query("UPDATE object_abuse SET nr_abuses='$abuse_points'" . " WHERE objectid='$objectid' AND source='$sourceip'"); - $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')"); + if ($points > 0) + { + $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')"); + } if ($abuse_points >= 6) { - echo " BLOCK IP adrress $sourceip on the firewall.\n"; + //echo " BLOCK IP adrress $sourceip on the firewall.\n"; $dbms->query("UPDATE object_abuse SET status='dropped'" . " WHERE objectid='$objectid' AND source='$sourceip'"); } @@ -715,10 +805,13 @@ function service_check() VALUES ('log_servicecheck', '0')"); } + echo "Running service check from log id $last_log.\n"; // Query the log-table + $log_limit = $last_log + BATCHSIZE; $qry = "SELECT logid, objectid, servicecode FROM log - WHERE logid > CAST(".$last_log." AS BIGINT) ORDER BY logid LIMIT " . BATCHSIZE; + WHERE logid > CAST(".$last_log." AS BIGINT) AND logid <= $log_limit + ORDER BY logid"; $log_res = $dbms->query($qry); //$log_res = $dbms->query("SELECT logid, objectid, servicecode,rawdata FROM log"); -- 2.11.0