-- DBA create the database and give access permissions.
--
-- $Log: create.sql,v $
--- Revision 1.19 2005-06-04 07:09:47 arjen
+-- Revision 1.20 2007-01-11 13:47:35 arjen
+-- Log_adv and derived tables removed.
+-- Create notifications from log entries with pattern matching.
+--
+-- Revision 1.19 2005/06/04 07:09:47 arjen
-- - Fixed field declaration for PostgreSQL 7.4
-- - New tables: log_abuse, object_abuse and service_pattern
-- - Added general service patterns in the service 'ANY'
--
--
-CREATE SEQUENCE "log_advid_seq";
-
-
-CREATE TABLE "log_adv"
-(
- "log_advid" bigint DEFAULT
- nextval('"log_advid_seq"'::text) NOT NULL,
- "logid" bigint NOT NULL,
- "detailed_table" text
-);
-
-
-CREATE INDEX log_adv_logid ON log_adv USING btree (logid);
-
-CREATE UNIQUE INDEX log_adv_log_advid ON log_adv USING btree (log_advid);
-
-SELECT setval ('"log_advid_seq"', 1, false);
-
-CREATE TABLE log_adv_kernel_network
-(
- device_in text,
- device_out text,
- hw_address text,
- source_ip INET,
- destination_ip INET,
- packet_length bigint,
- tos_bit text,
- prec_bit text,
- ttl int,
- header_id bigint,
- source_port int,
- destination_port int,
- body_length int,
- protocol text,
- body_len int,
- window text,
- urgp int,
- syn boolean DEFAULT false,
- type int,
- code int,
- sequence_number int,
- res text,
- rst boolean,
- df boolean
-) INHERITS (log_adv);
-
-CREATE TABLE log_adv_daemon
-(
- service TEXT,
- event TEXT
-) INHERITS (log_adv);
-
-CREATE INDEX log_adv_daemon_service ON log_adv_daemon (service);
-
-CREATE TABLE log_adv_daemon_email
-(
- source_ip INET,
- destination_ip INET,
- internal_messageid TEXT,
- external_messageid TEXT,
- to_email TEXT,
- from_email TEXT,
- delay interval,
- xdelay interval,
- mailer TEXT,
- status TEXT,
- pid INT,
- size INT,
- pri INT,
- relay TEXT,
- status_details TEXT,
- dsn TEXT
-) INHERITS (log_adv_daemon);
-
-CREATE INDEX log_adv_daemon_email_s_ip ON log_adv_daemon_email (source_ip);
-CREATE INDEX log_adv_daemon_email_d_ip ON log_adv_daemon_email (destination_ip);
-CREATE INDEX log_adv_daemon_email_to ON log_adv_daemon_email (to_email);
-CREATE INDEX log_adv_daemon_email_from ON log_adv_daemon_email (from_email);
-CREATE INDEX log_adv_email_status ON log_adv_daemon_email (status);
-
-
-/*
-CREATE TRUSTED PROCEDURAL LANGUAGE 'plpgsql' HANDLER "plpgsql_call_handler" LANCOMPILER '';
-CREATE FUNCTION "funct_processlog" () RETURNS opaque AS '
--- Initial date: September 18th 2002
--- Update: November 13th 2002
--- Author: Brenno J.S.A.A.F. de Winter
--- Abstract: This routine sets a flag
-DECLARE
- var_value VARCHAR;
- var_setting VARCHAR;
-
-BEGIN
-
- UPDATE db_value SET setting_value = ''TRUE'' WHERE setting = ''log_flag'';
- RETURN NULL;
-END;
- ' LANGUAGE 'plpgsql';
-
-
-CREATE TRIGGER "log_insert" AFTER INSERT ON "log" FOR EACH ROW EXECUTE PROCEDURE "funct_processlog" ();
-*/
---
---
-
CREATE TABLE "log_notification"
(
"notificationid" bigint,
source inet,
nr_abuses integer,
status text,
+ last_change timestamp,
primary key (objectid, source)
);
GRANT SELECT ON db_value TO GROUP view, GROUP ops, GROUP admin;
GRANT SELECT ON history TO GROUP view, GROUP ops, GROUP admin;
GRANT SELECT ON log TO GROUP view, GROUP ops, GROUP admin;
-GRANT SELECT ON log_adv TO GROUP view, GROUP ops, GROUP admin;
-GRANT SELECT ON log_adv_daemon TO GROUP view, GROUP ops, GROUP admin;
-GRANT SELECT ON log_adv_daemon_email TO GROUP view, GROUP ops, GROUP admin;
-GRANT SELECT ON log_adv_kernel_network TO GROUP view, GROUP ops, GROUP admin;
GRANT SELECT ON log_notification TO GROUP view, GROUP ops, GROUP admin;
GRANT SELECT ON notification TO GROUP view, GROUP ops, GROUP admin;
GRANT SELECT ON object TO GROUP view, GROUP ops, GROUP admin;
GRANT SELECT ON usr TO GROUP view, GROUP ops, GROUP admin;
GRANT INSERT ON action_user TO GROUP ops, GROUP admin;
+GRANT UPDATE ON action_user_actionstepid_seq TO GROUP ops, GROUP admin;
GRANT UPDATE ON notification TO GROUP ops, GROUP admin;
GRANT INSERT ON object TO GROUP admin;
GRANT UPDATE ON object TO GROUP ops, GROUP admin;
GRANT UPDATE ON usr TO GROUP view, GROUP ops, GROUP admin;
GRANT INSERT ON usr TO GROUP admin;
GRANT DELETE ON usr TO GROUP admin;
+GRANT SELECT ON service_pattern TO GROUP view, GROUP ops, GROUP admin;
+GRANT INSERT ON service_pattern TO GROUP admin;
+GRANT UPDATE ON service_pattern TO GROUP ops, GROUP admin;
+GRANT DELETE ON service_pattern TO GROUP admin;
+GRANT SELECT ON log_abuse TO GROUP view, GROUP ops, GROUP admin;
Gnucomo-0.0.8: September 4th 2003
$Log: gcm_daemon.php,v $
- Revision 1.20 2005-06-04 07:15:16 arjen
+ Revision 1.21 2007-01-11 13:47:41 arjen
+ Log_adv and derived tables removed.
+ Create notifications from log entries with pattern matching.
+
+ Revision 1.20 2005/06/04 07:15:16 arjen
Added pattern check on log entries with the service_pattern table.
Revision 1.19 2004/01/10 20:04:12 arjen
*/
-// $Id: gcm_daemon.php,v 1.20 2005-06-04 07:15:16 arjen Exp $
+// $Id: gcm_daemon.php,v 1.21 2007-01-11 13:47:41 arjen Exp $
ini_set('include_path', '.:./classes:../phpclasses');
ini_set('html_errors', 'false');
-define("BATCHSIZE", 200);
+define("BATCHSIZE", 2000);
//Tell the log that we're up.
define_syslog_variables();
// Set the standard variables //
+$purge_date =""; // Purge log entries until this date. Default: no purging
$project_name = "gnucomo"; // name of the entire project
$app_name = "gcm_daemon"; // name of the application running
$developrelease = "FALSE"; // Indicates if special debug settings are needed
-$db_version = 48; // The db_version indicates what the level of
+$db_version = 51; // The db_version indicates what the level of
// the database should be. If the database is
// old an update will be generated.
$gcmd_version = 5; // This value indicates the active version of
$project_name = $argv[$argi];
break;
+ case "-p":
+ $argi++;
+ $purge_date = $argv[$argi];
+ break;
+
default:
- echo "Usage: gcm_daemon [-c configname]\n";
+ echo "Usage: gcm_daemon [-c configname] [-p purgedate]\n";
exit();
break;
}
do
{
- //At this place we start processing new log-lines
+ if ($purge_date != "")
+ {
+ purge_old_logs($purge_date);
+ }
echo "Processing logs...\n";
process_log ();
service_check();
find_notifications();
- mail_notifications();
+ //mail_notifications();
// Gather the statistics for each object
syslog (LOG_INFO, "gcm_daemon ended nicely");
+function purge_old_logs($purge_date)
+{
+ global $dbms;
+
+ /*
+ * Make a temporary table with the logids of the old log entries
+ * We don't want to repeat a selection on the large log table itself.
+ */
+
+ echo "Purging log entries before $purge_date\n";
+
+ $dbms->query("CREATE TABLE gcm_deamon_old_log AS SELECT logid FROM log WHERE logid < $purge_date");
+ $dbms->query("SELECT logid FROM gcm_deamon_old_log");
+ echo $dbms->num_rows() . " log entries found.\n";
+ $r = $dbms->query("select notificationid from log_notification where logid in
+ (select logid from gcm_deamon_old_log) group by notificationid");
+ echo "Notifications that may be affected:\n";
+ $notifications = array();
+ for ($i=0; $i < $dbms->num_rows(); $i++)
+ {
+ $notif = $dbms->fetch_object($r, $i);
+ $notifications[] = $notif->notificationid;
+ echo $notif->notificationid . "\n";
+ }
+ $dbms->query("delete from log_notification where logid in
+ (select logid from gcm_deamon_old_log)");
+
+ // Clean up any notifications that have no more logs left
+ foreach ($notifications as $notif)
+ {
+ $c = $dbms->fetch_object($dbms->query("select count(*) from log_notification where notificationid=$notif"), 0);
+ echo "Notification $notif has " . $c->count . " log entries left.\n";
+ if ($c->count == 0)
+ {
+ echo "Cleaning up notification $notif.\n";
+ $dbms->query("delete from action_user where notificationid=$notif");
+ $dbms->query("delete from notification where notificationid=$notif");
+ }
+ }
+
+ $dbms->query("delete from log where logid in
+ (select logid from gcm_deamon_old_log)");
+
+ $dbms->query("drop table gcm_deamon_old_log");
+}
+
+
function process_log ()
{
echo "Last processed logid = $last_log \n";
//Query the log-table
+ $log_limit = $last_log + BATCHSIZE;
$local_sql = "SELECT * FROM log WHERE logid > CAST(".$last_log." AS BIGINT)
- ORDER BY logid LIMIT " . BATCHSIZE;
+ AND logid <= $log_limit ORDER BY logid";
$dbms->query($local_sql);
//Update the log-statistics in the object-table
$local_object_os = "";
$local_object_os_version = "";
+/*
while ($local_counter < $dbms->num_rows())
{
}
$local_counter++;
}
+*/
// Check for spam and other abuses.
$notifications = array();
- // notification: 'abuses exceeded'.
-
- $noqueue_res = $dbms->query("select logid, objectid, servicecode, rawdata from log where logid > " . $logstart
- . " order by logid limit " . BATCHSIZE);
+ $log_limit = $logstart + BATCHSIZE;
+ $noqueue_res = $dbms->query("select logid, objectid, servicecode, rawdata from log
+ where logid > $logstart and logid <= $log_limit
+ order by logid");
for ($row = 0; $row < $dbms->num_rows($noqueue_res); $row++)
{
$logentry = $dbms->fetch_object($noqueue_res, $row);
- echo "\n----------\n" . $logentry->rawdata . "\n----------\n";
+ //echo "\n----------\n" . $logentry->rawdata . "\n----------\n";
$service = $logentry->servicecode;
$pattern_res = $dbms->query("select * from service_pattern where service='$service'
OR service='ANY' order by rank");
// Scan the argument for '$n' expressions and expand
$srv_pat->argument = expand_arguments($srv_pat->argument, $matches);
- echo " " . $srv_pat->pattern . " matches.\n";
- echo " Matched string: " . $matches[0] . "\n";
- echo " Action = " . $srv_pat->action . "(" . $srv_pat->argument . ")\n\n";
+ //echo " " . $srv_pat->pattern . " matches.\n";
+ //echo " Matched string: " . $matches[0] . "\n";
+ //echo " Action = " . $srv_pat->action . "(" . $srv_pat->argument . ")\n\n";
$match_found = true;
switch ($srv_pat->action)
$notif = $srv_pat->argument;
if (!isset($notifications[$logentry->objectid][$notif]))
{
- echo "Creating notification $notif for object " . $logentry->objectid . ".\n";
+ //echo "Creating notification $notif for object " . $logentry->objectid . ".\n";
$remark = "Notification generated from Gnucomo pattern match.";
$notifications[$logentry->objectid][$notif] =
$dbms->new_notification($logentry->objectid, $notif, $remark);
}
if (isset($notifications[$logentry->objectid][$notif]))
{
- echo "Notification $notif for object " . $logentry->objectid . " already created.\n";
+ //echo "Notification $notif for object " . $logentry->objectid . " already created.\n";
$insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
$insertion .= $notifications[$logentry->objectid][$notif] . "', '";
$insertion .= $logentry->logid . "')";
{
echo "Creating notification $notif for object " . $logentry->objectid . ".\n";
$remark = "Abuses from IP address $source_ip exceeded the limit.";
- $notifications[$logentry->objectid][$notif][$source_ip] =
- $dbms->new_notification($logentry->objectid, $notif, $remark);
+ $notifid = $dbms->new_notification($logentry->objectid, $notif, $remark);
+ $notifications[$logentry->objectid][$notif][$source_ip] = $notifid;
+
+ // Add log entries from previously detected abuses
+
+ echo " Add log entries from previously detected abuses";
+ $abuses = $dbms->query("SELECT logid FROM log_abuse WHERE objectid = '" .
+ $logentry->objectid . "' AND source = '$source_ip'");
+ for ($abusenr = 0; $abusenr < $dbms->num_rows($abuses); $abusenr++)
+ {
+ $log_abuse = $dbms->fetch_object($abuses, $abusenr);
+ if ($log_abuse->logid != $logentry->logid)
+ {
+ $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
+ $insertion .= $notifid . "', '";
+ $insertion .= $log_abuse->logid . "')";
+ $dbms->query($insertion);
+ }
+ }
}
if (isset($notifications[$logentry->objectid][$notif][$source_ip]))
{
}
}
break;
+ case "forgive":
+ record_abuse($logentry->logid, $logentry->objectid, $srv_pat->argument, -4);
+ break;
default:
echo "Error: unrecognized action in service pattern: " . $srv_pat->action . "\n";
break;
}
else
{
- echo " " . $srv_pat->pattern . " does not match.\n";
+ // echo " " . $srv_pat->pattern . " does not match.\n";
}
}
$abuse_points = $points;
$ipaddress = gethostbyname($sourceip);
- echo " IP address for $sourceip is $ipaddress.\n";
+ //echo " IP address for $sourceip is $ipaddress.\n";
$sourceip = $ipaddress;
$abres = $dbms->query("SELECT * FROM object_abuse WHERE objectid='$objectid' AND source='$sourceip'");
- if (pg_numrows($abres) == 0)
+ if (pg_numrows($abres) == 0 && $points > 0)
{
- echo "$sourceip is new.\n";
+ //echo "$sourceip is new.\n";
$dbms->query("INSERT INTO object_abuse VALUES ('$objectid', '$sourceip', '$points')");
}
- else
+ else if (pg_numrows($abres) != 0)
{
$abuse = $dbms->fetch_object($abres, 0);
if ($abuse->status == '' || $abuse->status == 'dropped')
{
$abuse_points = $abuse->nr_abuses + $points;
- echo $sourceip . " will get " . $abuse_points . " abuse points, ";
- echo "Status was " . $abuse->status . "\n";
+ if ($abuse_points < 0)
+ {
+ $abuse_points = 0;
+ }
+ //echo $sourceip . " will get " . $abuse_points . " abuse points, ";
+ //echo "Status was " . $abuse->status . "\n";
$dbms->query("UPDATE object_abuse SET nr_abuses='$abuse_points'" .
" WHERE objectid='$objectid' AND source='$sourceip'");
- $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')");
+ if ($points > 0)
+ {
+ $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')");
+ }
if ($abuse_points >= 6)
{
- echo " BLOCK IP adrress $sourceip on the firewall.\n";
+ //echo " BLOCK IP adrress $sourceip on the firewall.\n";
$dbms->query("UPDATE object_abuse SET status='dropped'" .
" WHERE objectid='$objectid' AND source='$sourceip'");
}
VALUES ('log_servicecheck', '0')");
}
+ echo "Running service check from log id $last_log.\n";
// Query the log-table
+ $log_limit = $last_log + BATCHSIZE;
$qry = "SELECT logid, objectid, servicecode FROM log
- WHERE logid > CAST(".$last_log." AS BIGINT) ORDER BY logid LIMIT " . BATCHSIZE;
+ WHERE logid > CAST(".$last_log." AS BIGINT) AND logid <= $log_limit
+ ORDER BY logid";
$log_res = $dbms->query($qry);
//$log_res = $dbms->query("SELECT logid, objectid, servicecode,rawdata FROM log");
projects / gnucomo.git / commitdiff